feat(node): 연결 supervisor 및 CLI adaptors 개선
- runtimeSupervisor 단일 goroutine으로 연결·재연결 수명주기 관리 - 초기 연결 실패 시 fx OnStart 후크 중단 방지 (SDD S16) - finite 재연결 시도 고갈 시 노드 종료 - CLI adaptors: oneshot, opencode_sse, persistent 보강 및 테스트 확장
This commit is contained in:
parent
4dcf6f0cf9
commit
4a76851254
18 changed files with 1677 additions and 119 deletions
|
|
@ -39,9 +39,36 @@ func (c *CLI) executeCommand(ctx context.Context, spec runtime.ExecutionSpec, pr
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
|
// The context can be cancelled or time out before the process starts,
|
||||||
|
// which surfaces as a start error. Honor the cancellation contract with a
|
||||||
|
// single cancelled event and ErrRunCancelled instead of a start error.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
_ = sink.Emit(context.Background(), runtime.RuntimeEvent{
|
||||||
|
RunID: spec.RunID,
|
||||||
|
Type: runtime.EventTypeCancelled,
|
||||||
|
Message: cancelEventForContext(ctx.Err()),
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
})
|
||||||
|
return "", runtime.ErrRunCancelled
|
||||||
|
}
|
||||||
return "", emitReturnedError(ctx, sink, spec.RunID, fmt.Errorf("cli adapter: start %q: %w", profile.Command, err))
|
return "", emitReturnedError(ctx, sink, spec.RunID, fmt.Errorf("cli adapter: start %q: %w", profile.Command, err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Closing the stdout/stderr pipes on cancellation unblocks pending reads
|
||||||
|
// promptly. CommandContext only signals the direct child, so a forked
|
||||||
|
// grandchild that keeps the pipes open would otherwise stall the read until
|
||||||
|
// it exits on its own; closing the reader ends the read-vs-wait race.
|
||||||
|
readsDone := make(chan struct{})
|
||||||
|
defer close(readsDone)
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
_ = stdout.Close()
|
||||||
|
_ = stderr.Close()
|
||||||
|
case <-readsDone:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
||||||
RunID: spec.RunID,
|
RunID: spec.RunID,
|
||||||
Type: runtime.EventTypeStart,
|
Type: runtime.EventTypeStart,
|
||||||
|
|
@ -71,6 +98,19 @@ func (c *CLI) executeCommand(ctx context.Context, spec runtime.ExecutionSpec, pr
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
_ = <-stderrDone
|
_ = <-stderrDone
|
||||||
_ = cmd.Wait()
|
_ = cmd.Wait()
|
||||||
|
// A context cancellation can close the stdout pipe before cmd.Wait()
|
||||||
|
// observes the process exit, surfacing as a read error. Treat that as a
|
||||||
|
// cancellation so the user-cancel/timeout contract wins over the generic
|
||||||
|
// read-stdout error, emitting exactly one cancelled event.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
_ = sink.Emit(context.Background(), runtime.RuntimeEvent{
|
||||||
|
RunID: spec.RunID,
|
||||||
|
Type: runtime.EventTypeCancelled,
|
||||||
|
Message: cancelEventForContext(ctx.Err()),
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
})
|
||||||
|
return combinedOutput(), runtime.ErrRunCancelled
|
||||||
|
}
|
||||||
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
||||||
RunID: spec.RunID,
|
RunID: spec.RunID,
|
||||||
Type: runtime.EventTypeError,
|
Type: runtime.EventTypeError,
|
||||||
|
|
|
||||||
|
|
@ -513,9 +513,11 @@ func TestCLIExecuteOneShot_UserCancelEmitsUserCancelMessage(t *testing.T) {
|
||||||
cfg := config.CLIConf{
|
cfg := config.CLIConf{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
Profiles: map[string]config.CLIProfileConf{
|
Profiles: map[string]config.CLIProfileConf{
|
||||||
|
// Emit a marker before blocking so the test can cancel deterministically
|
||||||
|
// once the process is running and the stdout reader is active.
|
||||||
"slow-cancel": {
|
"slow-cancel": {
|
||||||
Command: "sh",
|
Command: "sh",
|
||||||
Args: []string{"-c", `sleep 10`},
|
Args: []string{"-c", `echo running; sleep 10`},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -523,33 +525,48 @@ func TestCLIExecuteOneShot_UserCancelEmitsUserCancelMessage(t *testing.T) {
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
go func() {
|
go func() {
|
||||||
time.Sleep(100 * time.Millisecond)
|
done <- c.Execute(ctx, noderuntime.ExecutionSpec{
|
||||||
cancel()
|
RunID: "run-usercancel",
|
||||||
|
Target: "slow-cancel",
|
||||||
|
Input: map[string]any{"prompt": "hello"},
|
||||||
|
}, sink)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
err := c.Execute(ctx, noderuntime.ExecutionSpec{
|
// Cancel only after the first stdout chunk proves the process started and the
|
||||||
RunID: "run-usercancel",
|
// read loop is active, so this exercises the stdout-read/command-wait
|
||||||
Target: "slow-cancel",
|
// cancellation race rather than a pre-start cancellation.
|
||||||
Input: map[string]any{"prompt": "hello"},
|
testutil.RequireEventually(t, 5*time.Second, 5*time.Millisecond, func() bool {
|
||||||
}, sink)
|
return testutil.CollectDeltas(sink.Events()) != ""
|
||||||
|
})
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
var err error
|
||||||
|
select {
|
||||||
|
case err = <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("execute did not return promptly after cancellation")
|
||||||
|
}
|
||||||
if err != noderuntime.ErrRunCancelled {
|
if err != noderuntime.ErrRunCancelled {
|
||||||
t.Fatalf("expected ErrRunCancelled, got %v", err)
|
t.Fatalf("expected ErrRunCancelled, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var cancelCount int
|
||||||
var lastMsg string
|
var lastMsg string
|
||||||
var cancelEvent bool
|
for _, ev := range sink.Events() {
|
||||||
events := sink.Events()
|
switch ev.Type {
|
||||||
if len(events) > 0 {
|
case noderuntime.EventTypeCancelled:
|
||||||
for i := len(events) - 1; i >= 0; i-- {
|
cancelCount++
|
||||||
if events[i].Type == noderuntime.EventTypeCancelled {
|
lastMsg = ev.Message
|
||||||
cancelEvent = true
|
case noderuntime.EventTypeError:
|
||||||
lastMsg = events[i].Message
|
t.Fatalf("expected no error event on cancellation, got %q", ev.Error)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !cancelEvent {
|
if cancelCount != 1 {
|
||||||
t.Fatal("expected cancelled event")
|
t.Fatalf("expected exactly one cancelled event, got %d", cancelCount)
|
||||||
}
|
}
|
||||||
if lastMsg != "user-cancel" {
|
if lastMsg != "user-cancel" {
|
||||||
t.Fatalf("expected cancel Message 'user-cancel', got %q", lastMsg)
|
t.Fatalf("expected cancel Message 'user-cancel', got %q", lastMsg)
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,24 @@ func (e *opencodeExecutor) Execute(ctx context.Context, spec runtime.ExecutionSp
|
||||||
|
|
||||||
prompt := extractPrompt(spec.Input)
|
prompt := extractPrompt(spec.Input)
|
||||||
if err := opencodePromptAsync(ctx, sess.serverURL, sess.sessionID, prompt, opts); err != nil {
|
if err := opencodePromptAsync(ctx, sess.serverURL, sess.sessionID, prompt, opts); err != nil {
|
||||||
|
// A cancellation while the async prompt POST is in flight surfaces as a
|
||||||
|
// request error; honor the cancellation contract with a bounded background
|
||||||
|
// abort on the remote session, a single cancelled event, and
|
||||||
|
// ErrRunCancelled instead of a generic prompt-async error.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
bg, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if sess.sessionID != "" {
|
||||||
|
_ = opencodeAbort(bg, sess.serverURL, sess.sessionID)
|
||||||
|
}
|
||||||
|
_ = sink.Emit(context.Background(), runtime.RuntimeEvent{
|
||||||
|
RunID: spec.RunID,
|
||||||
|
Type: runtime.EventTypeCancelled,
|
||||||
|
Message: cancelEventForContext(ctx.Err()),
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
})
|
||||||
|
return runtime.ErrRunCancelled
|
||||||
|
}
|
||||||
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
_ = sink.Emit(ctx, runtime.RuntimeEvent{
|
||||||
RunID: spec.RunID,
|
RunID: spec.RunID,
|
||||||
Type: runtime.EventTypeError,
|
Type: runtime.EventTypeError,
|
||||||
|
|
|
||||||
|
|
@ -31,13 +31,37 @@ type opencodeFakeServer struct {
|
||||||
createCalls atomic.Int32
|
createCalls atomic.Int32
|
||||||
lastPromptBody []byte
|
lastPromptBody []byte
|
||||||
createdID string
|
createdID string
|
||||||
|
|
||||||
|
// promptReady is signalled once the prompt_async handler has fully read the
|
||||||
|
// request body. Tests wait on this before issuing ordering decisions.
|
||||||
|
promptReady chan struct{}
|
||||||
|
// promptBlock, when non-nil, is drained once to unblock the prompt_async
|
||||||
|
// response write. nil means the handler replies immediately.
|
||||||
|
promptBlock *promptGate
|
||||||
|
}
|
||||||
|
|
||||||
|
// promptGate is a one-shot response gate for the prompt_async handler.
|
||||||
|
type promptGate struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
release chan struct{} // closed to unblock the handler
|
||||||
|
blocked chan struct{} // closed when the handler enters the blocked wait
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *promptGate) signalBlocked() {
|
||||||
|
select {
|
||||||
|
case <-g.blocked:
|
||||||
|
default:
|
||||||
|
close(g.blocked)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newOpencodeFakeServer(t *testing.T, sessionID string) (*opencodeFakeServer, *httptest.Server) {
|
func newOpencodeFakeServer(t *testing.T, sessionID string) (*opencodeFakeServer, *httptest.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
s := &opencodeFakeServer{
|
s := &opencodeFakeServer{
|
||||||
events: make(chan map[string]any, 32),
|
events: make(chan map[string]any, 32),
|
||||||
createdID: sessionID,
|
createdID: sessionID,
|
||||||
|
promptReady: make(chan struct{}, 1),
|
||||||
}
|
}
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
eventHandler := func(w http.ResponseWriter, r *http.Request) {
|
eventHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
@ -94,6 +118,22 @@ func newOpencodeFakeServer(t *testing.T, sessionID string) (*opencodeFakeServer,
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
s.lastPromptBody = append([]byte(nil), buf[:n]...)
|
s.lastPromptBody = append([]byte(nil), buf[:n]...)
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
|
// Signal that the prompt body has been received so callers can
|
||||||
|
// observe the acceptance event before the response is written.
|
||||||
|
select {
|
||||||
|
case s.promptReady <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
// If a response gate is armed, wait for release or request cancellation.
|
||||||
|
gate := s.currentPromptGate()
|
||||||
|
if gate != nil {
|
||||||
|
gate.signalBlocked()
|
||||||
|
select {
|
||||||
|
case <-gate.release:
|
||||||
|
case <-r.Context().Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
w.WriteHeader(http.StatusAccepted)
|
w.WriteHeader(http.StatusAccepted)
|
||||||
case "abort":
|
case "abort":
|
||||||
s.abortCalls.Add(1)
|
s.abortCalls.Add(1)
|
||||||
|
|
@ -127,6 +167,68 @@ func (s *opencodeFakeServer) push(ev map[string]any) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BlockPromptResponse arms the prompt response gate. The prompt_async handler
|
||||||
|
// will not reply with 202 until UnblockPromptResponse is called or the server
|
||||||
|
// closes. Safe to call multiple times; only the most recent gate is in effect.
|
||||||
|
// Returns the gate reference for handler-block ordering checks.
|
||||||
|
func (s *opencodeFakeServer) BlockPromptResponse() *promptGate {
|
||||||
|
s.mu.Lock()
|
||||||
|
g := &promptGate{release: make(chan struct{}), blocked: make(chan struct{})}
|
||||||
|
s.promptBlock = g
|
||||||
|
s.mu.Unlock()
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnblockPromptResponse releases the most recent prompt response gate. If no
|
||||||
|
// gate is armed, this is a no-op. Safe to call multiple times; only the
|
||||||
|
// first call closes the release channel.
|
||||||
|
func (s *opencodeFakeServer) UnblockPromptResponse() {
|
||||||
|
s.mu.Lock()
|
||||||
|
block := s.promptBlock
|
||||||
|
s.promptBlock = nil
|
||||||
|
s.mu.Unlock()
|
||||||
|
if block == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
block.mu.Lock()
|
||||||
|
if block.closed {
|
||||||
|
block.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
block.closed = true
|
||||||
|
block.mu.Unlock()
|
||||||
|
close(block.release)
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentPromptGate returns the currently armed gate, or nil.
|
||||||
|
func (s *opencodeFakeServer) currentPromptGate() *promptGate {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.promptBlock
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitPromptBodyRead waits until the handler has fully read the prompt body,
|
||||||
|
// or times out.
|
||||||
|
func waitPromptBodyRead(t *testing.T, fake *opencodeFakeServer) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case <-fake.promptReady:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("prompt body was not read by handler")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitPromptResponseBlocked waits until the handler has entered the blocked
|
||||||
|
// wait on the response gate, or times out.
|
||||||
|
func waitPromptResponseBlocked(t *testing.T, g *promptGate) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case <-g.blocked:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("handler did not enter blocked wait on response gate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func opencodeGlobalEvent(eventType string, props map[string]any) map[string]any {
|
func opencodeGlobalEvent(eventType string, props map[string]any) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
"directory": "/tmp/iop-test",
|
"directory": "/tmp/iop-test",
|
||||||
|
|
@ -779,3 +881,76 @@ func TestCLIExecuteOpencodeSSE_ContextCancelAbortsSession(t *testing.T) {
|
||||||
t.Fatal("expected /session/{id}/abort to be called")
|
t.Fatal("expected /session/{id}/abort to be called")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCLIExecuteOpencodeSSE_ContextCancelAfterPromptAcceptedAbortsSession
|
||||||
|
// verifies the regression: when prompt_async POST reaches the server (202) but
|
||||||
|
// the response is blocked, cancelling the run context must abort the remote
|
||||||
|
// session exactly once via /abort, emit exactly one cancelled event, and
|
||||||
|
// return ErrRunCancelled with no error event.
|
||||||
|
func TestCLIExecuteOpencodeSSE_ContextCancelAfterPromptAcceptedAbortsSession(t *testing.T) {
|
||||||
|
fake, srv := newOpencodeFakeServer(t, "ses_abort")
|
||||||
|
|
||||||
|
cfg := config.CLIConf{
|
||||||
|
Enabled: true,
|
||||||
|
Profiles: map[string]config.CLIProfileConf{
|
||||||
|
"opencode": opencodeSSEProfile(srv.URL),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c := clipkg.New(cfg, zap.NewNop())
|
||||||
|
sink := &testutil.FakeSink{}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
// Arm the response gate before Execute so the handler blocks on it.
|
||||||
|
blocked := fake.BlockPromptResponse()
|
||||||
|
defer fake.UnblockPromptResponse()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- c.Execute(ctx, noderuntime.ExecutionSpec{
|
||||||
|
RunID: "run-abort-after-accept",
|
||||||
|
Target: "opencode",
|
||||||
|
Input: map[string]any{"prompt": "abort-test"},
|
||||||
|
}, sink)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for the handler to confirm it has the body and is blocked on the gate.
|
||||||
|
waitPromptBodyRead(t, fake)
|
||||||
|
waitPromptResponseBlocked(t, blocked)
|
||||||
|
|
||||||
|
// Cancel the run context. The adapter must handle this by sending a
|
||||||
|
// bounded /abort, emitting one cancelled event, and returning ErrRunCancelled.
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
// The execute call must return ErrRunCancelled within a bounded window.
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != noderuntime.ErrRunCancelled {
|
||||||
|
t.Fatalf("expected ErrRunCancelled, got %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("execute did not return after prompt-accepted cancellation")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute returns only after the in-flight cancellation path completes /abort.
|
||||||
|
if got := fake.abortCalls.Load(); got != 1 {
|
||||||
|
t.Fatalf("expected /abort call count == 1 when Execute returned, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly one cancelled event, no error event.
|
||||||
|
var cancelCount, errCount int
|
||||||
|
for _, e := range sink.Events() {
|
||||||
|
switch e.Type {
|
||||||
|
case noderuntime.EventTypeCancelled:
|
||||||
|
cancelCount++
|
||||||
|
case noderuntime.EventTypeError:
|
||||||
|
errCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cancelCount != 1 {
|
||||||
|
t.Fatalf("expected exactly 1 cancelled event, got %d", cancelCount)
|
||||||
|
}
|
||||||
|
if errCount != 0 {
|
||||||
|
t.Fatalf("expected 0 error events, got %d", errCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -264,6 +264,18 @@ func (e *persistentExecutor) Execute(ctx context.Context, spec runtime.Execution
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := run.setup(); err != nil {
|
if err := run.setup(); err != nil {
|
||||||
|
// Writing the prompt aborts with the context error when the run is
|
||||||
|
// cancelled or times out mid-setup. Emit a single cancelled event and
|
||||||
|
// return ErrRunCancelled so cancellation is not reported as a setup error.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
_ = sink.Emit(context.Background(), runtime.RuntimeEvent{
|
||||||
|
RunID: spec.RunID,
|
||||||
|
Type: runtime.EventTypeCancelled,
|
||||||
|
Message: cancelEventForContext(ctx.Err()),
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
})
|
||||||
|
return runtime.ErrRunCancelled
|
||||||
|
}
|
||||||
return emitRuntimeError(ctx, sink, spec.RunID, err.Error())
|
return emitRuntimeError(ctx, sink, spec.RunID, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -210,7 +210,7 @@ func TestCLIExecutePersistent_CompletionMarkerLineEndsRun(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
@ -261,7 +261,7 @@ func TestCLIExecutePersistent_CompletionMarkerRegexEndsRun(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
@ -311,7 +311,7 @@ func TestCLIExecutePersistent_NoMarkerFallsBackToIdleTimeout(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
@ -361,7 +361,7 @@ func TestCLIExecutePersistent_IdleTimeoutMessageReason(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,7 @@ func TestCLIExecutePersistentWaitsForSlowFirstOutput(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,12 @@ func TestCLIExecutePersistentTerminalSendsCarriageReturn(t *testing.T) {
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 100,
|
ResponseIdleTimeoutMS: 100,
|
||||||
StartupIdleTimeoutMS: 50,
|
// This scenario asserts the raw "\r" terminator, so the prompt must be
|
||||||
|
// written only after the helper has entered raw mode. The readiness
|
||||||
|
// marker anchors the drain to that point; the timeout must exceed the
|
||||||
|
// re-exec'd helper's worst-case cold start (~700ms observed) so the
|
||||||
|
// drain never settles before the marker arrives.
|
||||||
|
StartupIdleTimeoutMS: 800,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -98,7 +103,10 @@ func TestCLIExecutePersistentTerminalSendsCarriageReturn(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = c.Stop(ctx) }()
|
defer func() { _ = c.Stop(ctx) }()
|
||||||
|
|
||||||
execCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
|
// Generous bound: the startup drain alone is 800ms (see profile) and process
|
||||||
|
// spawn is slow and variable in constrained CI, so a 1s cap would race the run
|
||||||
|
// to completion. This still asserts completion, not a timeout.
|
||||||
|
execCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
sink := &testutil.FakeSink{}
|
sink := &testutil.FakeSink{}
|
||||||
|
|
@ -136,7 +144,10 @@ func TestCLIExecutePersistentTerminalAcceptsClaudeBypassWarning(t *testing.T) {
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 100,
|
ResponseIdleTimeoutMS: 100,
|
||||||
StartupIdleTimeoutMS: 250,
|
// The startup drain must observe the bypass-warning screen (and
|
||||||
|
// auto-accept it) before the prompt is sent; cover the re-exec'd
|
||||||
|
// helper's worst-case cold start, which is larger under -race.
|
||||||
|
StartupIdleTimeoutMS: 800,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -182,7 +193,10 @@ func TestCLIExecutePersistentTerminalAcceptsClaudeWorkspaceTrustAndBypassWarning
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 100,
|
ResponseIdleTimeoutMS: 100,
|
||||||
StartupIdleTimeoutMS: 250,
|
// The startup drain must observe the trust + bypass-warning screens
|
||||||
|
// (and auto-accept them) before the prompt is sent; cover the
|
||||||
|
// re-exec'd helper's worst-case cold start, which is larger under -race.
|
||||||
|
StartupIdleTimeoutMS: 800,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -228,7 +242,9 @@ func TestCLIExecutePersistentClaudeTUICancelsSessionLimitUpgradePrompt(t *testin
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 500,
|
ResponseIdleTimeoutMS: 500,
|
||||||
StartupIdleTimeoutMS: 50,
|
// Cover the re-exec'd helper's variable cold start so the startup
|
||||||
|
// drain waits for its readiness marker before the prompt is sent.
|
||||||
|
StartupIdleTimeoutMS: 300,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -282,7 +298,10 @@ func TestCLIExecutePersistentClaudeTUIReplaysPromptAfterStartupReadyRace(t *test
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 100,
|
ResponseIdleTimeoutMS: 100,
|
||||||
StartupIdleTimeoutMS: 40,
|
// Cover the re-exec'd helper's variable cold start so the startup
|
||||||
|
// drain observes the trust screen and auto-accepts it before the
|
||||||
|
// prompt is sent, keeping the replay-after-ready path deterministic.
|
||||||
|
StartupIdleTimeoutMS: 300,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
@ -406,8 +425,12 @@ func TestCLIExecutePersistentClaudeTUIFiltersTerminalChrome(t *testing.T) {
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
Profiles: map[string]config.CLIProfileConf{
|
Profiles: map[string]config.CLIProfileConf{
|
||||||
"claude-tui": {
|
"claude-tui": {
|
||||||
Command: "sh",
|
Command: "sh",
|
||||||
Args: []string{"-c", `stty -echo; while IFS= read -r line; do printf '\033[2C\r\033[7A\342\227\217\033[1Cmessage:%s\r\n\302\267 Cogitating... (1s, 1 tokens)\r\n\342\235\257 ' "$line"; done`},
|
// The frame begins with cursor-home + erase-display, mirroring a real
|
||||||
|
// claude TUI redraw, so the rendered assistant screen is independent
|
||||||
|
// of whether the prompt raced `stty -echo` and got echoed: any echoed
|
||||||
|
// characters are erased before the assistant line is drawn.
|
||||||
|
Args: []string{"-c", `stty -echo; while IFS= read -r line; do printf '\033[H\033[2J\033[2C\r\033[7A\342\227\217\033[1Cmessage:%s\r\n\302\267 Cogitating... (1s, 1 tokens)\r\n\342\235\257 ' "$line"; done`},
|
||||||
Persistent: true,
|
Persistent: true,
|
||||||
Terminal: true,
|
Terminal: true,
|
||||||
ResponseIdleTimeoutMS: 100,
|
ResponseIdleTimeoutMS: 100,
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ import (
|
||||||
"os"
|
"os"
|
||||||
osexec "os/exec"
|
osexec "os/exec"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRawTUIHelperProcess(t *testing.T) {
|
func TestRawTUIHelperProcess(t *testing.T) {
|
||||||
|
|
@ -36,8 +35,20 @@ func setRawTerminal() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// emitStartupReady announces that the helper has finished configuring its raw
|
||||||
|
// terminal and is now reading input. Re-exec'd test binaries pay a variable
|
||||||
|
// cold-start cost, so the adapter's startup drain must observe this marker (and
|
||||||
|
// idle after it) before it writes the prompt. Otherwise the prompt would race
|
||||||
|
// terminal setup and be echoed or line-translated. The marker is consumed by
|
||||||
|
// the startup drain and never surfaces as a delta.
|
||||||
|
func emitStartupReady() {
|
||||||
|
fmt.Fprint(os.Stdout, "iop-tui-ready\r\n")
|
||||||
|
_ = os.Stdout.Sync()
|
||||||
|
}
|
||||||
|
|
||||||
func runRawTUIHelper() {
|
func runRawTUIHelper() {
|
||||||
setRawTerminal()
|
setRawTerminal()
|
||||||
|
emitStartupReady()
|
||||||
readPromptAndReply()
|
readPromptAndReply()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -114,10 +125,15 @@ func runClaudeTrustAndWarningHelper() {
|
||||||
|
|
||||||
func runClaudeRateLimitHelper() {
|
func runClaudeRateLimitHelper() {
|
||||||
setRawTerminal()
|
setRawTerminal()
|
||||||
|
emitStartupReady()
|
||||||
for {
|
for {
|
||||||
buf := make([]byte, 1)
|
buf := make([]byte, 1)
|
||||||
n, err := os.Stdin.Read(buf)
|
n, err := os.Stdin.Read(buf)
|
||||||
if n > 0 && buf[0] == '\r' {
|
// Trigger on either terminator: in raw mode the adapter's terminator is
|
||||||
|
// "\r", but if the prompt raced terminal setup the line discipline may have
|
||||||
|
// translated it to "\n". Accepting both keeps session-limit detection
|
||||||
|
// independent of the re-exec'd helper's cold-start timing.
|
||||||
|
if n > 0 && (buf[0] == '\r' || buf[0] == '\n') {
|
||||||
fmt.Fprint(os.Stdout, "\u25cf \u23bf You've hit your session limit \u00b7 resets 12:50pm (Asia/Seoul)\r\n")
|
fmt.Fprint(os.Stdout, "\u25cf \u23bf You've hit your session limit \u00b7 resets 12:50pm (Asia/Seoul)\r\n")
|
||||||
fmt.Fprint(os.Stdout, "What do you want to do?\r\n❯ 1. Stop and wait for limit to reset\r\n2. Upgrade your plan\r\n")
|
fmt.Fprint(os.Stdout, "What do you want to do?\r\n❯ 1. Stop and wait for limit to reset\r\n2. Upgrade your plan\r\n")
|
||||||
_ = os.Stdout.Sync()
|
_ = os.Stdout.Sync()
|
||||||
|
|
@ -130,17 +146,17 @@ func runClaudeRateLimitHelper() {
|
||||||
|
|
||||||
func runClaudeReadyReplayHelper() {
|
func runClaudeReadyReplayHelper() {
|
||||||
setRawTerminal()
|
setRawTerminal()
|
||||||
fmt.Fprint(os.Stdout, "Quick safety check: Is this a project you created or one you trust?\r\n❯ 1. Yes, I trust this folder\r\n2. No, exit\r\n")
|
emitStartupReady()
|
||||||
_ = os.Stdout.Sync()
|
|
||||||
|
|
||||||
_ = readRawLine()
|
// Handshake via blocking reads instead of a fixed sleep: swallow the first
|
||||||
time.Sleep(150 * time.Millisecond)
|
// prompt delivery to model a CLI that is not ready for input yet, then
|
||||||
_ = readRawLine()
|
// advertise the ready screen so the adapter replays the prompt exactly once.
|
||||||
|
_ = readRawLine() // first prompt, intentionally dropped before the ready screen
|
||||||
|
|
||||||
fmt.Fprint(os.Stdout, "\r\n❯ ")
|
fmt.Fprint(os.Stdout, "\r\n❯ ")
|
||||||
_ = os.Stdout.Sync()
|
_ = os.Stdout.Sync()
|
||||||
|
|
||||||
prompt := readRawLine()
|
prompt := readRawLine() // replayed prompt
|
||||||
fmt.Fprintf(os.Stdout, "\r\n\u25cf replay:%s\r\n\u276f ", prompt)
|
fmt.Fprintf(os.Stdout, "\r\n\u25cf replay:%s\r\n\u276f ", prompt)
|
||||||
_ = os.Stdout.Sync()
|
_ = os.Stdout.Sync()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,11 @@ import (
|
||||||
iop "iop/proto/gen/iop"
|
iop "iop/proto/gen/iop"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// readySignalTimeout bounds the post-handler NodeReadyRequest/ack round trip. A
|
||||||
|
// ready failure is retryable (not a *transport.ConnectError), so the supervisor
|
||||||
|
// reconnects rather than shutting the node down.
|
||||||
|
const readySignalTimeout = 10 * time.Second
|
||||||
|
|
||||||
// DialFunc is the signature used to connect to Edge. Can be replaced in tests.
|
// DialFunc is the signature used to connect to Edge. Can be replaced in tests.
|
||||||
type DialFunc func(ctx context.Context, addr, token string, logger *zap.Logger) (*transport.RegisterResult, error)
|
type DialFunc func(ctx context.Context, addr, token string, logger *zap.Logger) (*transport.RegisterResult, error)
|
||||||
|
|
||||||
|
|
@ -107,6 +112,17 @@ func connectRuntime(ctx context.Context, cfg *config.NodeConfig, logger *zap.Log
|
||||||
})
|
})
|
||||||
result.Session.SetHandler(n)
|
result.Session.SetHandler(n)
|
||||||
|
|
||||||
|
// The handler is installed; only now tell edge we are ready to receive
|
||||||
|
// dispatch. Edge opens run/tunnel/command eligibility and pumps any waiter
|
||||||
|
// stranded while this node was offline strictly in response to this signal, so
|
||||||
|
// the first request can never race ahead of the handler. A failed ready
|
||||||
|
// handshake is retryable: close the partial owner and let the supervisor
|
||||||
|
// reconnect rather than sit connected-but-unreachable.
|
||||||
|
if err := result.Session.SignalReady(readySignalTimeout); err != nil {
|
||||||
|
owner.close()
|
||||||
|
return nil, fmt.Errorf("signal ready: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
logger.Info("connected to edge",
|
logger.Info("connected to edge",
|
||||||
zap.String("node_id", result.NodeID),
|
zap.String("node_id", result.NodeID),
|
||||||
zap.String("alias", result.Alias),
|
zap.String("alias", result.Alias),
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
toki "git.toki-labs.com/toki/proto-socket/go"
|
toki "git.toki-labs.com/toki/proto-socket/go"
|
||||||
|
"git.toki-labs.com/toki/proto-socket/go/packets"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
|
|
@ -28,9 +29,48 @@ func bootstrapEdgeParserMap() toki.ParserMap {
|
||||||
m := &iop.RegisterRequest{}
|
m := &iop.RegisterRequest{}
|
||||||
return m, proto.Unmarshal(b, m)
|
return m, proto.Unmarshal(b, m)
|
||||||
},
|
},
|
||||||
|
toki.TypeNameOf(&iop.NodeReadyRequest{}): func(b []byte) (proto.Message, error) {
|
||||||
|
m := &iop.NodeReadyRequest{}
|
||||||
|
return m, proto.Unmarshal(b, m)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bindEdgeReadyResponder answers the node's post-handler NodeReadyRequest with a
|
||||||
|
// positive ack, so connectRuntime's SignalReady completes and the session is
|
||||||
|
// treated as fully established. Mock edges that accept registration must install
|
||||||
|
// it or the node's ready handshake times out and reconnects. When readyCh is
|
||||||
|
// non-nil it fires only after the ready response has been written to the socket.
|
||||||
|
// This lets a test safely close the edge side after a fully established session,
|
||||||
|
// instead of racing the in-flight ready handshake.
|
||||||
|
func bindEdgeReadyResponder(client *toki.TcpClient, readyCh chan<- struct{}) {
|
||||||
|
client.Communicator.AddRequestListener(
|
||||||
|
toki.TypeNameOf(&iop.NodeReadyRequest{}),
|
||||||
|
func(message proto.Message, requestNonce int32) {
|
||||||
|
if _, ok := message.(*iop.NodeReadyRequest); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data, err := proto.Marshal(&iop.NodeReadyResponse{Ready: true})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := client.QueuePacket(&packets.PacketBase{
|
||||||
|
TypeName: toki.TypeNameOf(&iop.NodeReadyResponse{}),
|
||||||
|
ResponseNonce: requestNonce,
|
||||||
|
Data: data,
|
||||||
|
}); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if readyCh != nil {
|
||||||
|
select {
|
||||||
|
case readyCh <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func freeAddrForBootstrap(t *testing.T) (host string, port int, addr string) {
|
func freeAddrForBootstrap(t *testing.T) (host string, port int, addr string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
|
@ -60,6 +100,453 @@ func useTempCwd(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startAcceptingEdge starts a mock edge bound to host:port that accepts every
|
||||||
|
// RegisterRequest. The returned channels fire once per registration and
|
||||||
|
// disconnect, respectively.
|
||||||
|
func startAcceptingEdge(t *testing.T, ctx context.Context, host string, port int, nodeID string) (chan struct{}, chan struct{}) {
|
||||||
|
t.Helper()
|
||||||
|
regCh := make(chan struct{}, 16)
|
||||||
|
disconnectCh := make(chan struct{}, 16)
|
||||||
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
|
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
||||||
|
client.AddDisconnectListener(func(_ *toki.TcpClient) {
|
||||||
|
disconnectCh <- struct{}{}
|
||||||
|
})
|
||||||
|
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
||||||
|
&client.Communicator,
|
||||||
|
func(_ *iop.RegisterRequest) (*iop.RegisterResponse, error) {
|
||||||
|
regCh <- struct{}{}
|
||||||
|
return &iop.RegisterResponse{
|
||||||
|
Accepted: true,
|
||||||
|
NodeId: nodeID,
|
||||||
|
Config: &iop.NodeConfigPayload{Runtime: &iop.NodeRuntimeConfig{Concurrency: 1}},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
bindEdgeReadyResponder(client, nil)
|
||||||
|
return client
|
||||||
|
})
|
||||||
|
if err := server.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start accepting edge: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { server.Stop() })
|
||||||
|
return regCh, disconnectCh
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRejectingEdge starts a mock edge bound to host:port that rejects every
|
||||||
|
// RegisterRequest with the given reason.
|
||||||
|
func startRejectingEdge(t *testing.T, ctx context.Context, host string, port int, reason string) {
|
||||||
|
t.Helper()
|
||||||
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
|
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
||||||
|
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
||||||
|
&client.Communicator,
|
||||||
|
func(_ *iop.RegisterRequest) (*iop.RegisterResponse, error) {
|
||||||
|
return &iop.RegisterResponse{Accepted: false, Reason: reason}, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return client
|
||||||
|
})
|
||||||
|
if err := server.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start rejecting edge: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { server.Stop() })
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorConnectsWhenEdgeStartsAfterNode verifies the actual DialEdge
|
||||||
|
// connection-refused path: Node starts first against a closed port, keeps
|
||||||
|
// retrying in the same process, and establishes exactly one session after Edge
|
||||||
|
// begins listening on that same port. SDD S16.
|
||||||
|
func TestSupervisorConnectsWhenEdgeStartsAfterNode(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
host, port, addr := freeAddrForBootstrap(t)
|
||||||
|
|
||||||
|
const refusedAttempts = 3
|
||||||
|
var dialCount int32
|
||||||
|
retryReady := make(chan struct{}, refusedAttempts)
|
||||||
|
releaseRetry := make(chan struct{})
|
||||||
|
observedDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
return transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
})
|
||||||
|
fakeSleeper := func(ctx context.Context, _ time.Duration) {
|
||||||
|
retryReady <- struct{}{}
|
||||||
|
if atomic.LoadInt32(&dialCount) >= refusedAttempts {
|
||||||
|
select {
|
||||||
|
case <-releaseRetry:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 0, IntervalSec: 1},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(observedDialer), bootstrap.WithSleeper(fakeSleeper)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
|
// Edge is not listening yet. Each completed sleep entry follows a real
|
||||||
|
// DialEdge connection-refused result. Hold the third retry until the listener
|
||||||
|
// is live so the next actual dial succeeds deterministically.
|
||||||
|
for attempt := 1; attempt <= refusedAttempts; attempt++ {
|
||||||
|
select {
|
||||||
|
case <-retryReady:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatalf("timeout waiting for refused DialEdge attempt %d", attempt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != refusedAttempts {
|
||||||
|
t.Fatalf("expected %d real refused dials before Edge startup, got %d", refusedAttempts, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
t.Cleanup(serverCancel)
|
||||||
|
regCh, _ := startAcceptingEdge(t, serverCtx, host, port, "delayed-node")
|
||||||
|
close(releaseRetry)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-regCh:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for delayed initial registration")
|
||||||
|
}
|
||||||
|
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != refusedAttempts+1 {
|
||||||
|
t.Fatalf("expected %d dials (3 real refusals + 1 success), got %d", refusedAttempts+1, count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly one session must be established.
|
||||||
|
select {
|
||||||
|
case <-regCh:
|
||||||
|
t.Fatal("unexpected second registration; supervisor should hold one session")
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorUnlimitedAttemptsPassesTen verifies that an explicit
|
||||||
|
// max_attempts=0 (unlimited) keeps retrying past the former hard-coded 10-attempt
|
||||||
|
// cap and eventually connects. SDD S16/S17.
|
||||||
|
func TestSupervisorUnlimitedAttemptsPassesTen(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
host, port, addr := freeAddrForBootstrap(t)
|
||||||
|
|
||||||
|
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
t.Cleanup(serverCancel)
|
||||||
|
regCh, _ := startAcceptingEdge(t, serverCtx, host, port, "unlimited-node")
|
||||||
|
|
||||||
|
const failFirst = 15 // exceeds the old hard-coded 10-attempt cap
|
||||||
|
var dialCount int32
|
||||||
|
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
if atomic.AddInt32(&dialCount, 1) <= failFirst {
|
||||||
|
return nil, errors.New("edge unavailable")
|
||||||
|
}
|
||||||
|
return transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
})
|
||||||
|
fakeSleeper := func(context.Context, time.Duration) {}
|
||||||
|
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 0, IntervalSec: 1},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(fakeDialer), bootstrap.WithSleeper(fakeSleeper)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-regCh:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for registration after >10 unlimited attempts")
|
||||||
|
}
|
||||||
|
|
||||||
|
count := atomic.LoadInt32(&dialCount)
|
||||||
|
if count != failFirst+1 {
|
||||||
|
t.Fatalf("expected %d dials, got %d", failFirst+1, count)
|
||||||
|
}
|
||||||
|
if count <= 10 {
|
||||||
|
t.Fatalf("unlimited policy must retry past 10 attempts, got %d", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorFiniteExhaustionExitsOne verifies that a finite reconnect policy
|
||||||
|
// makes exactly max_attempts initial-connect attempts and then terminates the
|
||||||
|
// node with exit code 1. SDD S17.
|
||||||
|
func TestSupervisorFiniteExhaustionExitsOne(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
_, _, addr := freeAddrForBootstrap(t) // no server listening
|
||||||
|
|
||||||
|
var dialCount int32
|
||||||
|
fakeDialer := bootstrap.DialFunc(func(_ context.Context, _, _ string, _ *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
return nil, errors.New("edge unavailable")
|
||||||
|
})
|
||||||
|
fakeSleeper := func(context.Context, time.Duration) {}
|
||||||
|
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 3, IntervalSec: 0},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(fakeDialer), bootstrap.WithSleeper(fakeSleeper)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case sig := <-app.Wait():
|
||||||
|
if sig.ExitCode != 1 {
|
||||||
|
t.Fatalf("expected exit code 1, got %d", sig.ExitCode)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for finite exhaustion shutdown")
|
||||||
|
}
|
||||||
|
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != 3 {
|
||||||
|
t.Fatalf("expected exactly 3 initial-connect attempts, got %d", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorFatalFailureDoesNotRetry verifies that an authoritative
|
||||||
|
// registration rejection is non-retryable: the supervisor exits with code 1
|
||||||
|
// after a single attempt without any retry. SDD S17.
|
||||||
|
func TestSupervisorFatalFailureDoesNotRetry(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
host, port, addr := freeAddrForBootstrap(t)
|
||||||
|
|
||||||
|
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
t.Cleanup(serverCancel)
|
||||||
|
startRejectingEdge(t, serverCtx, host, port, "bad-token")
|
||||||
|
|
||||||
|
var dialCount int32
|
||||||
|
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
return transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 5, IntervalSec: 0},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(fakeDialer)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case sig := <-app.Wait():
|
||||||
|
if sig.ExitCode != 1 {
|
||||||
|
t.Fatalf("expected exit code 1 on fatal failure, got %d", sig.ExitCode)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for fatal shutdown")
|
||||||
|
}
|
||||||
|
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != 1 {
|
||||||
|
t.Fatalf("fatal failure must not retry: expected 1 dial, got %d", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorShutdownCancelsRetry verifies that a local shutdown cancels a
|
||||||
|
// pending retry sleep, ends cleanly with no error, and runs no additional
|
||||||
|
// attempt. SDD S17.
|
||||||
|
func TestSupervisorShutdownCancelsRetry(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
_, _, addr := freeAddrForBootstrap(t) // no server listening
|
||||||
|
|
||||||
|
var dialCount int32
|
||||||
|
fakeDialer := bootstrap.DialFunc(func(_ context.Context, _, _ string, _ *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
return nil, errors.New("edge unavailable")
|
||||||
|
})
|
||||||
|
|
||||||
|
sleeping := make(chan struct{}, 1)
|
||||||
|
fakeSleeper := func(ctx context.Context, _ time.Duration) {
|
||||||
|
select {
|
||||||
|
case sleeping <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
<-ctx.Done() // block until the supervisor is cancelled by shutdown
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 0, IntervalSec: 1},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(fakeDialer), bootstrap.WithSleeper(fakeSleeper)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait until the supervisor has dialed once and parked in the retry sleep.
|
||||||
|
select {
|
||||||
|
case <-sleeping:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for supervisor to enter retry sleep")
|
||||||
|
}
|
||||||
|
countAtSleep := atomic.LoadInt32(&dialCount)
|
||||||
|
if countAtSleep != 1 {
|
||||||
|
t.Fatalf("expected exactly 1 dial before parking in sleep, got %d", countAtSleep)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local shutdown must cancel the sleep and stop cleanly with no error.
|
||||||
|
stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer stopCancel()
|
||||||
|
if err := app.Stop(stopCtx); err != nil {
|
||||||
|
t.Fatalf("app stop must be clean, got: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != countAtSleep {
|
||||||
|
t.Fatalf("shutdown must not trigger extra dial: had %d, got %d", countAtSleep, count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorShutdownWinsInFlightDialFatal fixes the ordering between a
|
||||||
|
// local stop and a dialer that observes cancellation but returns a fatal result
|
||||||
|
// afterward. Cancellation must produce a clean stop, no exit 1, and no retry.
|
||||||
|
func TestSupervisorShutdownWinsInFlightDialFatal(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
_, _, addr := freeAddrForBootstrap(t)
|
||||||
|
|
||||||
|
started := make(chan struct{}, 1)
|
||||||
|
var dialCount int32
|
||||||
|
dialer := bootstrap.DialFunc(func(ctx context.Context, _, _ string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
started <- struct{}{}
|
||||||
|
<-ctx.Done()
|
||||||
|
return transport.DialEdge(context.Background(), "127.0.0.1:0", "tok", logger)
|
||||||
|
})
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 0, IntervalSec: 1},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(dialer)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for in-flight dial")
|
||||||
|
}
|
||||||
|
|
||||||
|
stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer stopCancel()
|
||||||
|
if err := app.Stop(stopCtx); err != nil {
|
||||||
|
t.Fatalf("app stop must be clean, got: %v", err)
|
||||||
|
}
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != 1 {
|
||||||
|
t.Fatalf("shutdown must not retry fatal late result: got %d dials", count)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case sig := <-app.Wait():
|
||||||
|
t.Fatalf("shutdown must not request failure exit, got %+v", sig)
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSupervisorShutdownClosesInFlightDialSuccess verifies that a dialer which
|
||||||
|
// observes cancellation and then returns a successful registered session cannot
|
||||||
|
// install or leak that late owner. The one registered session is closed by the
|
||||||
|
// post-dial cancellation fence and no failure exit or retry is emitted.
|
||||||
|
func TestSupervisorShutdownClosesInFlightDialSuccess(t *testing.T) {
|
||||||
|
useTempCwd(t)
|
||||||
|
host, port, addr := freeAddrForBootstrap(t)
|
||||||
|
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
t.Cleanup(serverCancel)
|
||||||
|
regCh, disconnectCh := startAcceptingEdge(t, serverCtx, host, port, "late-success-node")
|
||||||
|
|
||||||
|
started := make(chan struct{}, 1)
|
||||||
|
var dialCount int32
|
||||||
|
dialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
atomic.AddInt32(&dialCount, 1)
|
||||||
|
started <- struct{}{}
|
||||||
|
<-ctx.Done()
|
||||||
|
lateCtx, lateCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer lateCancel()
|
||||||
|
return transport.DialEdge(lateCtx, edgeAddr, token, logger)
|
||||||
|
})
|
||||||
|
cfg := &config.NodeConfig{
|
||||||
|
Transport: config.TransportConf{EdgeAddr: addr, Token: "tok"},
|
||||||
|
Reconnect: config.ReconnectConf{MaxAttempts: 0, IntervalSec: 1},
|
||||||
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(dialer)), fx.NopLogger)
|
||||||
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer startCancel()
|
||||||
|
if err := app.Start(startCtx); err != nil {
|
||||||
|
t.Fatalf("app start: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for in-flight dial")
|
||||||
|
}
|
||||||
|
|
||||||
|
stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer stopCancel()
|
||||||
|
stopDone := make(chan error, 1)
|
||||||
|
go func() { stopDone <- app.Stop(stopCtx) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-regCh:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for late successful registration")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-stopDone:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("app stop must be clean, got: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for app stop")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-disconnectCh:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("late successful session leaked after supervisor shutdown")
|
||||||
|
}
|
||||||
|
if count := atomic.LoadInt32(&dialCount); count != 1 {
|
||||||
|
t.Fatalf("shutdown must not retry late success: got %d dials", count)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-regCh:
|
||||||
|
t.Fatal("unexpected second registration after shutdown")
|
||||||
|
case sig := <-app.Wait():
|
||||||
|
t.Fatalf("shutdown must not request failure exit, got %+v", sig)
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestReconnectSupervisorReestablishesSession verifies that after the edge-side
|
// TestReconnectSupervisorReestablishesSession verifies that after the edge-side
|
||||||
// client closes the connection, the supervisor reconnects and re-registers.
|
// client closes the connection, the supervisor reconnects and re-registers.
|
||||||
func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
||||||
|
|
@ -69,8 +556,11 @@ func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
||||||
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
serverCtx, serverCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
t.Cleanup(serverCancel)
|
t.Cleanup(serverCancel)
|
||||||
|
|
||||||
// regCh fires each time a RegisterRequest is handled.
|
// regCh fires each time a RegisterRequest is handled; readyCh fires when a
|
||||||
|
// connection completes its ready handshake so the test can close it only after
|
||||||
|
// the session is fully established.
|
||||||
regCh := make(chan struct{}, 8)
|
regCh := make(chan struct{}, 8)
|
||||||
|
readyCh := make(chan struct{}, 8)
|
||||||
acceptedCh := make(chan *toki.TcpClient, 8)
|
acceptedCh := make(chan *toki.TcpClient, 8)
|
||||||
|
|
||||||
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
|
|
@ -86,6 +576,7 @@ func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
||||||
}, nil
|
}, nil
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
bindEdgeReadyResponder(client, readyCh)
|
||||||
acceptedCh <- client
|
acceptedCh <- client
|
||||||
return client
|
return client
|
||||||
})
|
})
|
||||||
|
|
@ -100,7 +591,19 @@ func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
||||||
Logging: config.LoggingConf{Level: "error"},
|
Logging: config.LoggingConf{Level: "error"},
|
||||||
}
|
}
|
||||||
|
|
||||||
app := fx.New(bootstrap.Module(cfg), fx.NopLogger)
|
// Wrap the real dialer so the test can wait for the initial session to be
|
||||||
|
// fully established before inducing a disconnect (OnStart is non-blocking).
|
||||||
|
established := make(chan struct{}, 1)
|
||||||
|
var establishOnce sync.Once
|
||||||
|
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
|
res, err := transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
if err == nil {
|
||||||
|
establishOnce.Do(func() { established <- struct{}{} })
|
||||||
|
}
|
||||||
|
return res, err
|
||||||
|
})
|
||||||
|
|
||||||
|
app := fx.New(bootstrap.Module(cfg, bootstrap.WithDialer(fakeDialer)), fx.NopLogger)
|
||||||
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
startCtx, startCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer startCancel()
|
defer startCancel()
|
||||||
|
|
||||||
|
|
@ -109,18 +612,30 @@ func TestReconnectSupervisorReestablishesSession(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = app.Stop(context.Background()) }()
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
// Wait for first registration.
|
// Wait for first registration and full establishment.
|
||||||
select {
|
select {
|
||||||
case <-regCh:
|
case <-regCh:
|
||||||
case <-time.After(3 * time.Second):
|
case <-time.After(3 * time.Second):
|
||||||
t.Fatal("timeout waiting for first registration")
|
t.Fatal("timeout waiting for first registration")
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case <-established:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial session establishment")
|
||||||
|
}
|
||||||
var firstClient *toki.TcpClient
|
var firstClient *toki.TcpClient
|
||||||
select {
|
select {
|
||||||
case firstClient = <-acceptedCh:
|
case firstClient = <-acceptedCh:
|
||||||
case <-time.After(time.Second):
|
case <-time.After(time.Second):
|
||||||
t.Fatal("timeout waiting for first accepted client")
|
t.Fatal("timeout waiting for first accepted client")
|
||||||
}
|
}
|
||||||
|
// Wait for the ready handshake before closing so the disconnect is induced on a
|
||||||
|
// fully-established session, not mid-SignalReady.
|
||||||
|
select {
|
||||||
|
case <-readyCh:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial ready handshake")
|
||||||
|
}
|
||||||
|
|
||||||
// Force edge-side close → remote disconnect on node.
|
// Force edge-side close → remote disconnect on node.
|
||||||
_ = firstClient.Close()
|
_ = firstClient.Close()
|
||||||
|
|
@ -143,6 +658,7 @@ func TestReconnectSupervisorExhaustsRetries(t *testing.T) {
|
||||||
t.Cleanup(serverCancel)
|
t.Cleanup(serverCancel)
|
||||||
|
|
||||||
acceptedCh := make(chan *toki.TcpClient, 4)
|
acceptedCh := make(chan *toki.TcpClient, 4)
|
||||||
|
readyCh := make(chan struct{}, 4)
|
||||||
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
||||||
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
||||||
|
|
@ -155,6 +671,7 @@ func TestReconnectSupervisorExhaustsRetries(t *testing.T) {
|
||||||
}, nil
|
}, nil
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
bindEdgeReadyResponder(client, readyCh)
|
||||||
acceptedCh <- client
|
acceptedCh <- client
|
||||||
return client
|
return client
|
||||||
})
|
})
|
||||||
|
|
@ -164,10 +681,17 @@ func TestReconnectSupervisorExhaustsRetries(t *testing.T) {
|
||||||
t.Cleanup(func() { server.Stop() })
|
t.Cleanup(func() { server.Stop() })
|
||||||
|
|
||||||
// Fake dialer: first call uses real DialEdge; subsequent calls fail immediately.
|
// Fake dialer: first call uses real DialEdge; subsequent calls fail immediately.
|
||||||
|
// established fires once the initial session is fully established so the test
|
||||||
|
// can induce the disconnect without racing the initial registration.
|
||||||
|
established := make(chan struct{}, 1)
|
||||||
var dialCount int32
|
var dialCount int32
|
||||||
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
if atomic.AddInt32(&dialCount, 1) == 1 {
|
if atomic.AddInt32(&dialCount, 1) == 1 {
|
||||||
return transport.DialEdge(ctx, edgeAddr, token, logger)
|
res, err := transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
if err == nil {
|
||||||
|
established <- struct{}{}
|
||||||
|
}
|
||||||
|
return res, err
|
||||||
}
|
}
|
||||||
return nil, errors.New("fake reconnect failure")
|
return nil, errors.New("fake reconnect failure")
|
||||||
})
|
})
|
||||||
|
|
@ -191,13 +715,25 @@ func TestReconnectSupervisorExhaustsRetries(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = app.Stop(context.Background()) }()
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
// Wait for first connection then close it from the edge side.
|
// Wait for the initial session to establish, then close it from the edge side.
|
||||||
|
select {
|
||||||
|
case <-established:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial session establishment")
|
||||||
|
}
|
||||||
var firstClient *toki.TcpClient
|
var firstClient *toki.TcpClient
|
||||||
select {
|
select {
|
||||||
case firstClient = <-acceptedCh:
|
case firstClient = <-acceptedCh:
|
||||||
case <-time.After(3 * time.Second):
|
case <-time.After(3 * time.Second):
|
||||||
t.Fatal("timeout waiting for first connection")
|
t.Fatal("timeout waiting for first connection")
|
||||||
}
|
}
|
||||||
|
// Close only after the ready handshake so the first session counts as an
|
||||||
|
// established connection, not a mid-SignalReady failure.
|
||||||
|
select {
|
||||||
|
case <-readyCh:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial ready handshake")
|
||||||
|
}
|
||||||
_ = firstClient.Close()
|
_ = firstClient.Close()
|
||||||
|
|
||||||
// Supervisor exhausts retries → Shutdown(ExitCode(1)).
|
// Supervisor exhausts retries → Shutdown(ExitCode(1)).
|
||||||
|
|
@ -227,6 +763,7 @@ func TestReconnectSupervisorPolicyTimingAndLimit(t *testing.T) {
|
||||||
t.Cleanup(serverCancel)
|
t.Cleanup(serverCancel)
|
||||||
|
|
||||||
acceptedCh := make(chan *toki.TcpClient, 4)
|
acceptedCh := make(chan *toki.TcpClient, 4)
|
||||||
|
readyCh := make(chan struct{}, 4)
|
||||||
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
client := toki.NewTcpClient(conn, 30, 10, bootstrapEdgeParserMap())
|
||||||
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
||||||
|
|
@ -239,6 +776,7 @@ func TestReconnectSupervisorPolicyTimingAndLimit(t *testing.T) {
|
||||||
}, nil
|
}, nil
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
bindEdgeReadyResponder(client, readyCh)
|
||||||
acceptedCh <- client
|
acceptedCh <- client
|
||||||
return client
|
return client
|
||||||
})
|
})
|
||||||
|
|
@ -248,10 +786,17 @@ func TestReconnectSupervisorPolicyTimingAndLimit(t *testing.T) {
|
||||||
t.Cleanup(func() { server.Stop() })
|
t.Cleanup(func() { server.Stop() })
|
||||||
|
|
||||||
// Fake dialer: first call uses real DialEdge; subsequent calls fail immediately.
|
// Fake dialer: first call uses real DialEdge; subsequent calls fail immediately.
|
||||||
|
// established fires once the initial session is fully established so the test
|
||||||
|
// can induce the disconnect without racing the initial registration.
|
||||||
|
established := make(chan struct{}, 1)
|
||||||
var dialCount int32
|
var dialCount int32
|
||||||
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
fakeDialer := bootstrap.DialFunc(func(ctx context.Context, edgeAddr, token string, logger *zap.Logger) (*transport.RegisterResult, error) {
|
||||||
if atomic.AddInt32(&dialCount, 1) == 1 {
|
if atomic.AddInt32(&dialCount, 1) == 1 {
|
||||||
return transport.DialEdge(ctx, edgeAddr, token, logger)
|
res, err := transport.DialEdge(ctx, edgeAddr, token, logger)
|
||||||
|
if err == nil {
|
||||||
|
established <- struct{}{}
|
||||||
|
}
|
||||||
|
return res, err
|
||||||
}
|
}
|
||||||
return nil, errors.New("fake reconnect failure")
|
return nil, errors.New("fake reconnect failure")
|
||||||
})
|
})
|
||||||
|
|
@ -284,13 +829,25 @@ func TestReconnectSupervisorPolicyTimingAndLimit(t *testing.T) {
|
||||||
}
|
}
|
||||||
defer func() { _ = app.Stop(context.Background()) }()
|
defer func() { _ = app.Stop(context.Background()) }()
|
||||||
|
|
||||||
// Wait for first connection then close it from the edge side.
|
// Wait for the initial session to establish, then close it from the edge side.
|
||||||
|
select {
|
||||||
|
case <-established:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial session establishment")
|
||||||
|
}
|
||||||
var firstClient *toki.TcpClient
|
var firstClient *toki.TcpClient
|
||||||
select {
|
select {
|
||||||
case firstClient = <-acceptedCh:
|
case firstClient = <-acceptedCh:
|
||||||
case <-time.After(3 * time.Second):
|
case <-time.After(3 * time.Second):
|
||||||
t.Fatal("timeout waiting for first connection")
|
t.Fatal("timeout waiting for first connection")
|
||||||
}
|
}
|
||||||
|
// Close only after the ready handshake so the first session counts as an
|
||||||
|
// established connection, not a mid-SignalReady failure.
|
||||||
|
select {
|
||||||
|
case <-readyCh:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("timeout waiting for initial ready handshake")
|
||||||
|
}
|
||||||
_ = firstClient.Close()
|
_ = firstClient.Close()
|
||||||
|
|
||||||
// Supervisor exhausts all 10 retries → Shutdown(ExitCode(1)).
|
// Supervisor exhausts all 10 retries → Shutdown(ExitCode(1)).
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@ package bootstrap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|
@ -14,10 +13,11 @@ import (
|
||||||
"iop/packages/go/observability"
|
"iop/packages/go/observability"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runtimeSupervisor owns the node's single active edge connection and the
|
// runtimeSupervisor owns the node's single active edge connection across its
|
||||||
// reconnect loop. It is created once per fx lifecycle and drives
|
// entire lifetime. A single supervisor goroutine serially performs the initial
|
||||||
// connect/reconnect/shutdown. The mutex guards current across the reconnect
|
// dial, waits for the established session to end, and reconnects, so at most one
|
||||||
// goroutine and the OnStop hook.
|
// in-flight dial and one active session exist at any time. The mutex guards
|
||||||
|
// current across the supervisor goroutine and the OnStop hook.
|
||||||
type runtimeSupervisor struct {
|
type runtimeSupervisor struct {
|
||||||
cfg *config.NodeConfig
|
cfg *config.NodeConfig
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
|
|
@ -28,25 +28,22 @@ type runtimeSupervisor struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
current *runtimeOwner
|
current *runtimeOwner
|
||||||
cancelSupervisor context.CancelFunc
|
cancelSupervisor context.CancelFunc
|
||||||
|
done chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// start establishes the initial connection, launches the reconnect supervisor
|
// start launches the metrics server and the single supervisor goroutine, then
|
||||||
// goroutine, and starts the metrics server. It maps a connect failure to the
|
// returns immediately. Establishing the initial connection is the supervisor's
|
||||||
// same "bootstrap: %w" error the OnStart hook returned before extraction.
|
// responsibility, so a transient Edge outage at startup no longer fails the fx
|
||||||
func (s *runtimeSupervisor) start(ctx context.Context) error {
|
// OnStart hook (SDD S16).
|
||||||
owner, err := connectRuntime(ctx, s.cfg, s.logger, s.dialer)
|
func (s *runtimeSupervisor) start(_ context.Context) error {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("bootstrap: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.mu.Lock()
|
|
||||||
s.current = owner
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
supCtx, cancel := context.WithCancel(context.Background())
|
supCtx, cancel := context.WithCancel(context.Background())
|
||||||
s.cancelSupervisor = cancel
|
s.cancelSupervisor = cancel
|
||||||
|
s.done = make(chan struct{})
|
||||||
|
|
||||||
go s.run(supCtx, owner.sess)
|
go func() {
|
||||||
|
defer close(s.done)
|
||||||
|
s.run(supCtx)
|
||||||
|
}()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := observability.ServeMetrics(s.cfg.Metrics.Port); err != nil {
|
if err := observability.ServeMetrics(s.cfg.Metrics.Port); err != nil {
|
||||||
|
|
@ -56,84 +53,107 @@ func (s *runtimeSupervisor) start(ctx context.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// run waits for the active session to end, then drives a reconnect. It exits on
|
// run drives the full connectivity lifecycle: connect (initial), wait for the
|
||||||
// supervisor cancellation, a local shutdown, or when reconnect returns nil
|
// active session to end, then reconnect, repeating until the supervisor is
|
||||||
// (cancelled or exhausted).
|
// cancelled, a fatal error occurs, finite attempts are exhausted, or a local
|
||||||
func (s *runtimeSupervisor) run(supCtx context.Context, sess *transport.Session) {
|
// shutdown ends the session.
|
||||||
|
func (s *runtimeSupervisor) run(supCtx context.Context) {
|
||||||
|
owner := s.connect(supCtx, true)
|
||||||
|
if owner == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-supCtx.Done():
|
case <-supCtx.Done():
|
||||||
return
|
return
|
||||||
case <-sess.Done():
|
case <-owner.sess.Done():
|
||||||
}
|
}
|
||||||
|
|
||||||
// Guard against context cancellation racing with disconnect.
|
// Guard against supervisor cancellation racing with disconnect.
|
||||||
select {
|
select {
|
||||||
case <-supCtx.Done():
|
case <-supCtx.Done():
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if sess.IsLocalShutdown() {
|
if owner.sess.IsLocalShutdown() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
newOwner := s.reconnect(supCtx)
|
owner = s.connect(supCtx, false)
|
||||||
if newOwner == nil {
|
if owner == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sess = newOwner.sess
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reconnect runs the bounded reconnect loop. On success it swaps in the new
|
// connect runs the bounded connect/retry loop shared by the initial dial and
|
||||||
// owner and returns it. On supervisor cancellation it returns nil without
|
// established-session reconnect. On success it installs the new owner and
|
||||||
// changing state. On exhaustion it closes the current owner, requests node
|
// returns it. It returns nil after supervisor cancellation, a fatal
|
||||||
// shutdown, and returns nil. The attempt/interval defaults, sleep, dial timeout,
|
// (non-retryable) failure, or finite attempt exhaustion; the latter two request
|
||||||
// and logging match the prior inline loop.
|
// a node shutdown with exit code 1 before returning.
|
||||||
func (s *runtimeSupervisor) reconnect(supCtx context.Context) *runtimeOwner {
|
//
|
||||||
|
// The reconnect path sleeps before every attempt; the initial path issues its
|
||||||
|
// first dial immediately for fast startup and sleeps only before retries. A
|
||||||
|
// finite policy (max_attempts>0) makes exactly max_attempts attempts before
|
||||||
|
// exhausting; an unlimited policy (max_attempts=0) retries until cancelled.
|
||||||
|
func (s *runtimeSupervisor) connect(supCtx context.Context, initial bool) *runtimeOwner {
|
||||||
maxAttempts := s.cfg.Reconnect.MaxAttempts
|
maxAttempts := s.cfg.Reconnect.MaxAttempts
|
||||||
if maxAttempts <= 0 {
|
|
||||||
maxAttempts = 10
|
|
||||||
}
|
|
||||||
intervalSec := s.cfg.Reconnect.IntervalSec
|
intervalSec := s.cfg.Reconnect.IntervalSec
|
||||||
if intervalSec < 0 {
|
unlimited := maxAttempts == 0
|
||||||
intervalSec = 10
|
|
||||||
}
|
|
||||||
|
|
||||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
for attempt := 1; unlimited || attempt <= maxAttempts; attempt++ {
|
||||||
select {
|
if attempt > 1 || !initial {
|
||||||
case <-supCtx.Done():
|
s.sleeper(supCtx, time.Duration(intervalSec)*time.Second)
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.sleeper(supCtx, time.Duration(intervalSec)*time.Second)
|
|
||||||
if supCtx.Err() != nil {
|
if supCtx.Err() != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("reconnecting to edge",
|
s.logger.Info("connecting to edge",
|
||||||
|
zap.Bool("initial", initial),
|
||||||
zap.Int("attempt", attempt),
|
zap.Int("attempt", attempt),
|
||||||
zap.Int("max_attempts", maxAttempts),
|
zap.Int("max_attempts", maxAttempts),
|
||||||
|
zap.Bool("unlimited", unlimited),
|
||||||
zap.Int("interval_sec", intervalSec),
|
zap.Int("interval_sec", intervalSec),
|
||||||
)
|
)
|
||||||
|
|
||||||
dialCtx, dialCancel := context.WithTimeout(supCtx, 30*time.Second)
|
dialCtx, dialCancel := context.WithTimeout(supCtx, 30*time.Second)
|
||||||
o, dialErr := connectRuntime(dialCtx, s.cfg, s.logger, s.dialer)
|
owner, err := connectRuntime(dialCtx, s.cfg, s.logger, s.dialer)
|
||||||
dialCancel()
|
dialCancel()
|
||||||
if dialErr == nil {
|
// Local shutdown wins when a dial completes concurrently with supervisor
|
||||||
s.swapOwner(o)
|
// cancellation. A dialer may observe cancellation and still return a late
|
||||||
return o
|
// success or fatal result; never install that owner or turn it into exit 1.
|
||||||
|
if supCtx.Err() != nil {
|
||||||
|
if owner != nil {
|
||||||
|
owner.close()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
s.swapOwner(owner)
|
||||||
|
return owner
|
||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Warn("reconnect attempt failed",
|
if transport.IsFatalConnectError(err) {
|
||||||
|
s.fatal(err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Warn("connect attempt failed",
|
||||||
|
zap.Bool("initial", initial),
|
||||||
zap.Int("attempt", attempt),
|
zap.Int("attempt", attempt),
|
||||||
zap.Int("max_attempts", maxAttempts),
|
zap.Int("max_attempts", maxAttempts),
|
||||||
zap.Error(dialErr),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A local shutdown that lands on the final finite attempt falls out of the
|
||||||
|
// loop; treat it as cancellation rather than exhaustion.
|
||||||
|
if supCtx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
s.exhaust(maxAttempts)
|
s.exhaust(maxAttempts)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -149,9 +169,28 @@ func (s *runtimeSupervisor) swapOwner(owner *runtimeOwner) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fatal clears the current owner, logs the non-retryable failure, and requests a
|
||||||
|
// node shutdown with a non-zero exit code.
|
||||||
|
func (s *runtimeSupervisor) fatal(err error) {
|
||||||
|
s.clearCurrent()
|
||||||
|
s.logger.Error("connect failed with non-retryable error, shutting down node",
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
_ = s.shutdowner.Shutdown(fx.ExitCode(1))
|
||||||
|
}
|
||||||
|
|
||||||
// exhaust clears the current owner, logs the exhaustion, and requests a node
|
// exhaust clears the current owner, logs the exhaustion, and requests a node
|
||||||
// shutdown with a non-zero exit code.
|
// shutdown with a non-zero exit code.
|
||||||
func (s *runtimeSupervisor) exhaust(maxAttempts int) {
|
func (s *runtimeSupervisor) exhaust(maxAttempts int) {
|
||||||
|
s.clearCurrent()
|
||||||
|
s.logger.Error("reconnect exhausted, shutting down node",
|
||||||
|
zap.Int("max_attempts", maxAttempts),
|
||||||
|
)
|
||||||
|
_ = s.shutdowner.Shutdown(fx.ExitCode(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearCurrent detaches and closes the current owner, if any.
|
||||||
|
func (s *runtimeSupervisor) clearCurrent() {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
prev := s.current
|
prev := s.current
|
||||||
s.current = nil
|
s.current = nil
|
||||||
|
|
@ -159,22 +198,17 @@ func (s *runtimeSupervisor) exhaust(maxAttempts int) {
|
||||||
if prev != nil {
|
if prev != nil {
|
||||||
prev.close()
|
prev.close()
|
||||||
}
|
}
|
||||||
s.logger.Error("reconnect exhausted, shutting down node",
|
|
||||||
zap.Int("max_attempts", maxAttempts),
|
|
||||||
)
|
|
||||||
_ = s.shutdowner.Shutdown(fx.ExitCode(1))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// stop cancels the supervisor goroutine and closes the current connection.
|
// stop cancels the supervisor goroutine, waits for it to exit, and closes the
|
||||||
|
// current connection. Waiting guarantees no retry sleep or dial survives the
|
||||||
|
// shutdown and no additional attempt runs afterward.
|
||||||
func (s *runtimeSupervisor) stop() {
|
func (s *runtimeSupervisor) stop() {
|
||||||
if s.cancelSupervisor != nil {
|
if s.cancelSupervisor != nil {
|
||||||
s.cancelSupervisor()
|
s.cancelSupervisor()
|
||||||
}
|
}
|
||||||
s.mu.Lock()
|
if s.done != nil {
|
||||||
owner := s.current
|
<-s.done
|
||||||
s.current = nil
|
|
||||||
s.mu.Unlock()
|
|
||||||
if owner != nil {
|
|
||||||
owner.close()
|
|
||||||
}
|
}
|
||||||
|
s.clearCurrent()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package transport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
@ -13,6 +14,60 @@ import (
|
||||||
iop "iop/proto/gen/iop"
|
iop "iop/proto/gen/iop"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// connectFailureClass classifies a DialEdge failure so the connectivity
|
||||||
|
// supervisor can distinguish transient failures it should retry from fatal ones
|
||||||
|
// that must terminate the Node.
|
||||||
|
type connectFailureClass int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// connectFailureRetryable marks a transient failure (dial refused, network
|
||||||
|
// unreachable, timeout, transient register transport failure) that the
|
||||||
|
// supervisor should retry under the reconnect policy.
|
||||||
|
connectFailureRetryable connectFailureClass = iota
|
||||||
|
// connectFailureFatal marks a non-retryable failure (invalid address or
|
||||||
|
// authoritative registration rejection) that retrying cannot recover.
|
||||||
|
connectFailureFatal
|
||||||
|
)
|
||||||
|
|
||||||
|
// ConnectError wraps a DialEdge failure with a retry classification while
|
||||||
|
// preserving the underlying cause for errors.Is/errors.As and any transport
|
||||||
|
// status it carries. The supervisor consumes the classification through
|
||||||
|
// IsFatalConnectError rather than comparing error strings.
|
||||||
|
type ConnectError struct {
|
||||||
|
class connectFailureClass
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns the wrapped cause's message unchanged.
|
||||||
|
func (e *ConnectError) Error() string {
|
||||||
|
if e.cause == nil {
|
||||||
|
return "transport: connect error"
|
||||||
|
}
|
||||||
|
return e.cause.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unwrap exposes the original cause so errors.Is/errors.As reach it.
|
||||||
|
func (e *ConnectError) Unwrap() error { return e.cause }
|
||||||
|
|
||||||
|
// Retryable reports whether the failure is a transient one the supervisor
|
||||||
|
// should retry.
|
||||||
|
func (e *ConnectError) Retryable() bool { return e.class == connectFailureRetryable }
|
||||||
|
|
||||||
|
func wrapConnectError(class connectFailureClass, cause error) *ConnectError {
|
||||||
|
return &ConnectError{class: class, cause: cause}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsFatalConnectError reports whether err (or any wrapped cause) is a fatal,
|
||||||
|
// non-retryable connect failure. Errors that are not classified ConnectErrors
|
||||||
|
// are treated as retryable transient failures.
|
||||||
|
func IsFatalConnectError(err error) bool {
|
||||||
|
var ce *ConnectError
|
||||||
|
if errors.As(err, &ce) {
|
||||||
|
return !ce.Retryable()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
heartbeatIntervalSec = 2
|
heartbeatIntervalSec = 2
|
||||||
// heartbeatWaitSec is kept above heartbeatIntervalSec as defence in depth:
|
// heartbeatWaitSec is kept above heartbeatIntervalSec as defence in depth:
|
||||||
|
|
@ -56,28 +111,39 @@ type RegisterResult struct {
|
||||||
func DialEdge(ctx context.Context, addr, token string, logger *zap.Logger) (*RegisterResult, error) {
|
func DialEdge(ctx context.Context, addr, token string, logger *zap.Logger) (*RegisterResult, error) {
|
||||||
host, portStr, err := net.SplitHostPort(addr)
|
host, portStr, err := net.SplitHostPort(addr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("transport: invalid addr %q: %w", addr, err)
|
// Local address preparation error: retrying cannot recover it.
|
||||||
|
return nil, wrapConnectError(connectFailureFatal, fmt.Errorf("transport: invalid addr %q: %w", addr, err))
|
||||||
}
|
}
|
||||||
port, err := strconv.Atoi(portStr)
|
port, err := strconv.Atoi(portStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("transport: invalid port %q: %w", portStr, err)
|
return nil, wrapConnectError(connectFailureFatal, fmt.Errorf("transport: invalid port %q: %w", portStr, err))
|
||||||
|
}
|
||||||
|
if port < 1 || port > 65535 {
|
||||||
|
// A numeric port outside the TCP range is still a local configuration
|
||||||
|
// error. Preserve a standard range cause so callers can inspect it with
|
||||||
|
// errors.Is/errors.As instead of relying on the message.
|
||||||
|
rangeErr := &strconv.NumError{Func: "Atoi", Num: portStr, Err: strconv.ErrRange}
|
||||||
|
return nil, wrapConnectError(connectFailureFatal, fmt.Errorf("transport: invalid port %q: %w", portStr, rangeErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
dialer := net.Dialer{KeepAlive: 15 * time.Second}
|
dialer := net.Dialer{KeepAlive: 15 * time.Second}
|
||||||
conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("transport: dial edge %s: %w", addr, err)
|
// Edge unavailable / network transient failure: retryable.
|
||||||
|
return nil, wrapConnectError(connectFailureRetryable, fmt.Errorf("transport: dial edge %s: %w", addr, err))
|
||||||
}
|
}
|
||||||
client := toki.NewTcpClient(&writeDeadlineConn{Conn: conn, timeout: tcpWriteTimeout}, heartbeatIntervalSec, heartbeatWaitSec, nodeParserMap())
|
client := toki.NewTcpClient(&writeDeadlineConn{Conn: conn, timeout: tcpWriteTimeout}, heartbeatIntervalSec, heartbeatWaitSec, nodeParserMap())
|
||||||
|
|
||||||
resp, err := registerWithEdge(ctx, client, token, logger)
|
resp, err := registerWithEdge(ctx, client, token, logger)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
return nil, fmt.Errorf("transport: register: %w", err)
|
// Register request transport failure or context cancellation: retryable.
|
||||||
|
return nil, wrapConnectError(connectFailureRetryable, fmt.Errorf("transport: register: %w", err))
|
||||||
}
|
}
|
||||||
if !resp.GetAccepted() {
|
if !resp.GetAccepted() {
|
||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
return nil, fmt.Errorf("transport: register rejected: %s", resp.GetReason())
|
// Authoritative registration rejection (bad token/credential): fatal.
|
||||||
|
return nil, wrapConnectError(connectFailureFatal, fmt.Errorf("transport: register rejected: %s", resp.GetReason()))
|
||||||
}
|
}
|
||||||
|
|
||||||
sess := newSession(client, logger, resp.GetNodeId(), resp.GetAlias())
|
sess := newSession(client, logger, resp.GetNodeId(), resp.GetAlias())
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,19 @@
|
||||||
package transport
|
package transport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
toki "git.toki-labs.com/toki/proto-socket/go"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
|
|
||||||
|
iop "iop/proto/gen/iop"
|
||||||
)
|
)
|
||||||
|
|
||||||
type recordingConn struct {
|
type recordingConn struct {
|
||||||
|
|
@ -27,6 +37,166 @@ type noopAddr string
|
||||||
func (a noopAddr) Network() string { return string(a) }
|
func (a noopAddr) Network() string { return string(a) }
|
||||||
func (a noopAddr) String() string { return string(a) }
|
func (a noopAddr) String() string { return string(a) }
|
||||||
|
|
||||||
|
// closedLocalAddr returns a loopback address that has no listener, so a dial to
|
||||||
|
// it is refused deterministically.
|
||||||
|
func closedLocalAddr(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listen: %v", err)
|
||||||
|
}
|
||||||
|
addr := l.Addr().String()
|
||||||
|
_ = l.Close()
|
||||||
|
return addr
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectingEdgeParserMap() toki.ParserMap {
|
||||||
|
return toki.ParserMap{
|
||||||
|
toki.TypeNameOf(&iop.RegisterRequest{}): func(b []byte) (proto.Message, error) {
|
||||||
|
m := &iop.RegisterRequest{}
|
||||||
|
return m, proto.Unmarshal(b, m)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRejectingEdge starts a mock edge that answers RegisterRequest with
|
||||||
|
// Accepted=false and the given reason.
|
||||||
|
func startRejectingEdge(t *testing.T, ctx context.Context, reason string) string {
|
||||||
|
t.Helper()
|
||||||
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listen: %v", err)
|
||||||
|
}
|
||||||
|
addr := l.Addr().String()
|
||||||
|
host, portStr, _ := net.SplitHostPort(addr)
|
||||||
|
port, _ := strconv.Atoi(portStr)
|
||||||
|
_ = l.Close()
|
||||||
|
|
||||||
|
server := toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
|
||||||
|
client := toki.NewTcpClient(conn, 30, 10, rejectingEdgeParserMap())
|
||||||
|
toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
|
||||||
|
&client.Communicator,
|
||||||
|
func(_ *iop.RegisterRequest) (*iop.RegisterResponse, error) {
|
||||||
|
return &iop.RegisterResponse{Accepted: false, Reason: reason}, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return client
|
||||||
|
})
|
||||||
|
if err := server.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start rejecting edge: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { server.Stop() })
|
||||||
|
return addr
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDialEdgeClassifiesConnectFailures verifies that DialEdge classifies its
|
||||||
|
// failures into fatal (invalid address, authoritative registration rejection)
|
||||||
|
// and retryable (Edge unavailable) while preserving the underlying cause. REFACTOR-2.
|
||||||
|
func TestDialEdgeClassifiesConnectFailures(t *testing.T) {
|
||||||
|
logger := zap.NewNop()
|
||||||
|
|
||||||
|
t.Run("invalid address is fatal", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err := DialEdge(ctx, "missing-port", "tok", logger)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for invalid address")
|
||||||
|
}
|
||||||
|
if !IsFatalConnectError(err) {
|
||||||
|
t.Fatalf("invalid address must be fatal, got retryable: %v", err)
|
||||||
|
}
|
||||||
|
var ce *ConnectError
|
||||||
|
if !errors.As(err, &ce) || ce.Retryable() {
|
||||||
|
t.Fatalf("expected non-retryable ConnectError, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-numeric port is fatal", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err := DialEdge(ctx, "127.0.0.1:not-a-port", "tok", logger)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for invalid port")
|
||||||
|
}
|
||||||
|
if !IsFatalConnectError(err) {
|
||||||
|
t.Fatalf("invalid port must be fatal, got retryable: %v", err)
|
||||||
|
}
|
||||||
|
var numErr *strconv.NumError
|
||||||
|
if !errors.As(err, &numErr) || !errors.Is(err, strconv.ErrSyntax) {
|
||||||
|
t.Fatalf("expected preserved strconv syntax cause, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, port := range []string{"0", "-1", "65536"} {
|
||||||
|
port := port
|
||||||
|
t.Run("out-of-range port "+port+" is fatal", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err := DialEdge(ctx, net.JoinHostPort("127.0.0.1", port), "tok", logger)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error for out-of-range port %s", port)
|
||||||
|
}
|
||||||
|
if !IsFatalConnectError(err) {
|
||||||
|
t.Fatalf("out-of-range port %s must be fatal, got retryable: %v", port, err)
|
||||||
|
}
|
||||||
|
var ce *ConnectError
|
||||||
|
if !errors.As(err, &ce) || ce.Retryable() {
|
||||||
|
t.Fatalf("expected non-retryable ConnectError, got %v", err)
|
||||||
|
}
|
||||||
|
var numErr *strconv.NumError
|
||||||
|
if !errors.As(err, &numErr) || !errors.Is(err, strconv.ErrRange) {
|
||||||
|
t.Fatalf("expected preserved strconv range cause, got %v", err)
|
||||||
|
}
|
||||||
|
if numErr.Num != port {
|
||||||
|
t.Fatalf("range cause port = %q, want %q", numErr.Num, port)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("connection refused is retryable and preserves cause", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err := DialEdge(ctx, closedLocalAddr(t), "tok", logger)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for refused connection")
|
||||||
|
}
|
||||||
|
if IsFatalConnectError(err) {
|
||||||
|
t.Fatalf("refused connection must be retryable, got fatal: %v", err)
|
||||||
|
}
|
||||||
|
var ce *ConnectError
|
||||||
|
if !errors.As(err, &ce) || !ce.Retryable() {
|
||||||
|
t.Fatalf("expected retryable ConnectError, got %v", err)
|
||||||
|
}
|
||||||
|
// The underlying network error must remain reachable through the wrap.
|
||||||
|
var opErr *net.OpError
|
||||||
|
if !errors.As(err, &opErr) {
|
||||||
|
t.Fatalf("expected wrapped *net.OpError cause, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("registration rejected is fatal and preserves reason", func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
addr := startRejectingEdge(t, ctx, "bad-token")
|
||||||
|
_, err := DialEdge(ctx, addr, "tok", logger)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for rejected registration")
|
||||||
|
}
|
||||||
|
if !IsFatalConnectError(err) {
|
||||||
|
t.Fatalf("rejected registration must be fatal, got retryable: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "bad-token") {
|
||||||
|
t.Fatalf("expected rejection reason preserved in error, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unclassified error defaults to retryable", func(t *testing.T) {
|
||||||
|
if IsFatalConnectError(errors.New("plain")) {
|
||||||
|
t.Fatal("unclassified error must not be fatal")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriteDeadlineConnSetsAndClearsDeadline(t *testing.T) {
|
func TestWriteDeadlineConnSetsAndClearsDeadline(t *testing.T) {
|
||||||
base := &recordingConn{}
|
base := &recordingConn{}
|
||||||
conn := &writeDeadlineConn{Conn: base, timeout: 25 * time.Millisecond}
|
conn := &writeDeadlineConn{Conn: base, timeout: 25 * time.Millisecond}
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,10 @@ func nodeParserMap() toki.ParserMap {
|
||||||
m := &iop.RegisterResponse{}
|
m := &iop.RegisterResponse{}
|
||||||
return m, proto.Unmarshal(b, m)
|
return m, proto.Unmarshal(b, m)
|
||||||
},
|
},
|
||||||
|
toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
|
||||||
|
m := &iop.NodeReadyResponse{}
|
||||||
|
return m, proto.Unmarshal(b, m)
|
||||||
|
},
|
||||||
toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
|
toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
|
||||||
m := &iop.NodeConfigRefreshRequest{}
|
m := &iop.NodeConfigRefreshRequest{}
|
||||||
return m, proto.Unmarshal(b, m)
|
return m, proto.Unmarshal(b, m)
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,9 @@ package transport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
toki "git.toki-labs.com/toki/proto-socket/go"
|
toki "git.toki-labs.com/toki/proto-socket/go"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
@ -154,6 +156,28 @@ func (s *Session) SetHandler(h Handler) {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SignalReady tells edge this node has applied the config from RegisterResponse
|
||||||
|
// and installed its message handler, so edge may now open dispatch eligibility
|
||||||
|
// and pump any waiters stranded while the node was offline. It MUST be called
|
||||||
|
// after SetHandler: edge dispatches run/tunnel requests in response to this
|
||||||
|
// signal, and a request arriving before the handler is installed would be
|
||||||
|
// dropped. A non-ready ack (stale connection) or a transport error is returned so
|
||||||
|
// the caller tears the session down and lets the supervisor reconnect.
|
||||||
|
func (s *Session) SignalReady(timeout time.Duration) error {
|
||||||
|
resp, err := toki.SendRequestTyped[*iop.NodeReadyRequest, *iop.NodeReadyResponse](
|
||||||
|
&s.client.Communicator,
|
||||||
|
&iop.NodeReadyRequest{NodeId: s.nodeID},
|
||||||
|
timeout,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !resp.GetReady() {
|
||||||
|
return fmt.Errorf("edge rejected ready signal: %s", resp.GetReason())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// NodeID returns the session's node ID.
|
// NodeID returns the session's node ID.
|
||||||
func (s *Session) NodeID() string {
|
func (s *Session) NodeID() string {
|
||||||
return s.nodeID
|
return s.nodeID
|
||||||
|
|
|
||||||
|
|
@ -287,3 +287,55 @@ func TestSessionProviderTunnelRequest_NilAndErrHandler(t *testing.T) {
|
||||||
sess.Close()
|
sess.Close()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSessionSignalReady verifies the node→edge dispatch-ready handshake:
|
||||||
|
// SignalReady sends a NodeReadyRequest carrying the session's node id and returns
|
||||||
|
// nil only when the edge acks ready, and an error when the edge rejects the
|
||||||
|
// signal (stale connection) so the caller can tear down and reconnect.
|
||||||
|
func TestSessionSignalReady(t *testing.T) {
|
||||||
|
newReadyPipe := func(t *testing.T, ready bool, reason string) *transport.Session {
|
||||||
|
t.Helper()
|
||||||
|
edgeConn, nodeConn := net.Pipe()
|
||||||
|
edgeParser := toki.ParserMap{
|
||||||
|
toki.TypeNameOf(&iop.NodeReadyRequest{}): func(b []byte) (proto.Message, error) {
|
||||||
|
m := &iop.NodeReadyRequest{}
|
||||||
|
return m, proto.Unmarshal(b, m)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
nodeParser := toki.ParserMap{
|
||||||
|
toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
|
||||||
|
m := &iop.NodeReadyResponse{}
|
||||||
|
return m, proto.Unmarshal(b, m)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
edgeSide := toki.NewTcpClient(edgeConn, 0, 0, edgeParser)
|
||||||
|
nodeSide := toki.NewTcpClient(nodeConn, 0, 0, nodeParser)
|
||||||
|
t.Cleanup(func() { edgeSide.Close(); nodeSide.Close() })
|
||||||
|
|
||||||
|
toki.AddRequestListenerTyped[*iop.NodeReadyRequest, *iop.NodeReadyResponse](
|
||||||
|
&edgeSide.Communicator,
|
||||||
|
func(req *iop.NodeReadyRequest) (*iop.NodeReadyResponse, error) {
|
||||||
|
if req.GetNodeId() != "node-ready-test" {
|
||||||
|
return &iop.NodeReadyResponse{Ready: false, Reason: "unexpected node id"}, nil
|
||||||
|
}
|
||||||
|
return &iop.NodeReadyResponse{Ready: ready, Reason: reason}, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return transport.ExportNewSession(nodeSide, zap.NewNop(), "node-ready-test", "alias")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("ready ack succeeds", func(t *testing.T) {
|
||||||
|
sess := newReadyPipe(t, true, "")
|
||||||
|
if err := sess.SignalReady(2 * time.Second); err != nil {
|
||||||
|
t.Fatalf("SignalReady on ready ack: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-ready ack errors", func(t *testing.T) {
|
||||||
|
sess := newReadyPipe(t, false, "superseded")
|
||||||
|
err := sess.SignalReady(2 * time.Second)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("SignalReady must error when edge rejects the ready signal")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
||||||
334
scripts/dev/edge-node-reconnect-diagnostic.sh
Executable file
334
scripts/dev/edge-node-reconnect-diagnostic.sh
Executable file
|
|
@ -0,0 +1,334 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# edge-node-reconnect-diagnostic.sh - repo-internal diagnostic for edge/node reconnect
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||||
|
|
||||||
|
echo "[diagnostic] Starting edge-node-reconnect-diagnostic (repo-internal)..."
|
||||||
|
|
||||||
|
TMP_DIR=$(mktemp -d /tmp/iop-reconnect-diag-XXXXXX)
|
||||||
|
|
||||||
|
kill_process_tree() {
|
||||||
|
local pid="$1"
|
||||||
|
local child
|
||||||
|
if [ -z "$pid" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if command -v pgrep >/dev/null 2>&1; then
|
||||||
|
while IFS= read -r child; do
|
||||||
|
if [ -n "$child" ]; then
|
||||||
|
kill_process_tree "$child"
|
||||||
|
fi
|
||||||
|
done <<EOF_CHILDREN
|
||||||
|
$(pgrep -P "$pid" 2>/dev/null || true)
|
||||||
|
EOF_CHILDREN
|
||||||
|
fi
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
set +e
|
||||||
|
echo "[diagnostic] Cleaning up..."
|
||||||
|
kill_process_tree "${NODE_PID:-}"
|
||||||
|
kill_process_tree "${EDGE_PID:-}"
|
||||||
|
rm -rf "$TMP_DIR" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
EDGE_CONFIG="$TMP_DIR/edge.yaml"
|
||||||
|
NODE_CONFIG="$TMP_DIR/node.yaml"
|
||||||
|
EDGE_OUT="$TMP_DIR/edge.log"
|
||||||
|
NODE_OUT="$TMP_DIR/node.log"
|
||||||
|
CONSOLE_FIFO="$TMP_DIR/console.fifo"
|
||||||
|
|
||||||
|
# Registration wait ceiling; honor the dev override used by the verification command.
|
||||||
|
BIND_TIMEOUT="${IOP_DEV_RECONNECT_BIND_TIMEOUT:-45}"
|
||||||
|
|
||||||
|
PORT=$((30000 + RANDOM % 10000))
|
||||||
|
BOOTSTRAP_PORT=$((20000 + RANDOM % 10000))
|
||||||
|
EDGE_METRICS_PORT=$((40000 + RANDOM % 10000))
|
||||||
|
NODE_METRICS_PORT=$((50000 + RANDOM % 10000))
|
||||||
|
|
||||||
|
# Create fake-cli.sh. The Node's own [node-message] transcript below is the
|
||||||
|
# node-side payload record used for the Edge relay comparison.
|
||||||
|
MOCK_CLI="$TMP_DIR/fake-cli.sh"
|
||||||
|
cat <<'EOF' > "$MOCK_CLI"
|
||||||
|
#!/usr/bin/env sh
|
||||||
|
while IFS= read -r line; do
|
||||||
|
token=$(printf '%s\n' "$line" | sed -nE 's/.*(IOP_E2E_[A-Z0-9_]+).*/\1/p' | head -n 1)
|
||||||
|
if [ -n "$token" ]; then
|
||||||
|
printf '%s\n%s_TAIL\n' "$token" "$token"
|
||||||
|
else
|
||||||
|
printf 'IOP_E2E_UNKNOWN\n'
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
EOF
|
||||||
|
chmod +x "$MOCK_CLI"
|
||||||
|
|
||||||
|
# Create edge config
|
||||||
|
cat <<EOF > "$EDGE_CONFIG"
|
||||||
|
server:
|
||||||
|
listen: "127.0.0.1:$PORT"
|
||||||
|
metrics:
|
||||||
|
port: $EDGE_METRICS_PORT
|
||||||
|
bootstrap:
|
||||||
|
listen: "127.0.0.1:$BOOTSTRAP_PORT"
|
||||||
|
artifact_dir: "$TMP_DIR/artifacts"
|
||||||
|
nodes:
|
||||||
|
- id: test-node
|
||||||
|
alias: test-node
|
||||||
|
token: test-token
|
||||||
|
adapters:
|
||||||
|
cli:
|
||||||
|
enabled: true
|
||||||
|
profiles:
|
||||||
|
fake-cli:
|
||||||
|
command: "$MOCK_CLI"
|
||||||
|
persistent: true
|
||||||
|
response_idle_timeout_ms: 1000
|
||||||
|
console:
|
||||||
|
adapter: cli
|
||||||
|
target: fake-cli
|
||||||
|
session_id: default
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Create node config
|
||||||
|
cat <<EOF > "$NODE_CONFIG"
|
||||||
|
transport:
|
||||||
|
edge_addr: "127.0.0.1:$PORT"
|
||||||
|
token: test-token
|
||||||
|
reconnect:
|
||||||
|
interval_sec: 1
|
||||||
|
max_attempts: 0
|
||||||
|
metrics:
|
||||||
|
port: $NODE_METRICS_PORT
|
||||||
|
node:
|
||||||
|
id: test-node
|
||||||
|
alias: test-node
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Make console FIFO
|
||||||
|
mkfifo "$CONSOLE_FIFO"
|
||||||
|
|
||||||
|
# 1. Start edge
|
||||||
|
echo "[diagnostic] Starting edge.sh..."
|
||||||
|
IOP_EDGE_CONFIG="$EDGE_CONFIG" "$REPO_ROOT/scripts/dev/edge.sh" < "$CONSOLE_FIFO" > "$EDGE_OUT" 2>&1 &
|
||||||
|
EDGE_PID=$!
|
||||||
|
|
||||||
|
# Open FIFO for writing (blocks until edge reads it)
|
||||||
|
exec 3> "$CONSOLE_FIFO"
|
||||||
|
|
||||||
|
# 2. Start node
|
||||||
|
echo "[diagnostic] Starting node.sh..."
|
||||||
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" > "$NODE_OUT" 2>&1 &
|
||||||
|
NODE_PID=$!
|
||||||
|
|
||||||
|
# Wait for node registration
|
||||||
|
echo "[diagnostic] Awaiting node registration..."
|
||||||
|
deadline=$((SECONDS + BIND_TIMEOUT))
|
||||||
|
while ! grep -q 'connected reason="registered"' "$EDGE_OUT" 2>/dev/null; do
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "[diagnostic] Timeout waiting for node registration" >&2
|
||||||
|
cat "$EDGE_OUT" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
echo "[diagnostic] Node registered"
|
||||||
|
|
||||||
|
# Wait for registration to settle
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Send /nodes command
|
||||||
|
echo "/nodes" >&3
|
||||||
|
sleep 0.5
|
||||||
|
|
||||||
|
# Send message 1
|
||||||
|
echo "Convert token IOP_E2E_HELLO_BASIC and reply only with converted token" >&3
|
||||||
|
# Wait for message 1 complete
|
||||||
|
deadline=$((SECONDS + 15))
|
||||||
|
while ! grep -q "complete run_id=" "$EDGE_OUT" 2>/dev/null; do
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "[diagnostic] Timeout waiting for message 1 completion" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
echo "[diagnostic] Message 1 completed"
|
||||||
|
|
||||||
|
# Send message 2
|
||||||
|
# Save baseline of edge log count to check next completion
|
||||||
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
||||||
|
echo "Convert token IOP_E2E_HELLO_FORMAL and reply only with converted token" >&3
|
||||||
|
# Wait for message 2 complete (new lines only)
|
||||||
|
deadline=$((SECONDS + 15))
|
||||||
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "[diagnostic] Timeout waiting for message 2 completion" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
echo "[diagnostic] Message 2 completed"
|
||||||
|
|
||||||
|
# Send commands
|
||||||
|
echo "/capabilities" >&3
|
||||||
|
sleep 0.2
|
||||||
|
echo "/transport" >&3
|
||||||
|
sleep 0.2
|
||||||
|
echo "/sessions" >&3
|
||||||
|
sleep 0.2
|
||||||
|
echo "/terminate-session" >&3
|
||||||
|
sleep 0.5
|
||||||
|
|
||||||
|
# Restart Node
|
||||||
|
echo "[diagnostic] Killing node for reconnect test..."
|
||||||
|
kill_process_tree "$NODE_PID"
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
||||||
|
|
||||||
|
echo "[diagnostic] Restarting node..."
|
||||||
|
IOP_NODE_CONFIG="$NODE_CONFIG" "$REPO_ROOT/scripts/dev/node.sh" >> "$NODE_OUT" 2>&1 &
|
||||||
|
NODE_PID=$!
|
||||||
|
|
||||||
|
# Wait for re-registration (new lines only)
|
||||||
|
deadline=$((SECONDS + BIND_TIMEOUT))
|
||||||
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -E 'connected reason="registered"|node ready' 2>/dev/null; do
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "[diagnostic] Timeout waiting for node reconnect" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
echo "[diagnostic] Node reconnected"
|
||||||
|
|
||||||
|
# Wait for reconnect to settle
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Send message 3 post-reconnect
|
||||||
|
EDGE_BASELINE=$(wc -l < "$EDGE_OUT" | tr -d ' ')
|
||||||
|
echo "Convert token IOP_E2E_PING_BASIC and reply only with converted token" >&3
|
||||||
|
# Wait for message 3 complete
|
||||||
|
deadline=$((SECONDS + 10))
|
||||||
|
while ! tail -n +"$((EDGE_BASELINE + 1))" "$EDGE_OUT" | grep -q "complete run_id=" 2>/dev/null; do
|
||||||
|
if (( SECONDS >= deadline )); then
|
||||||
|
echo "[diagnostic] Timeout waiting for message 3 completion" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
echo "[diagnostic] Message 3 completed"
|
||||||
|
|
||||||
|
# Exit edge console
|
||||||
|
echo "/exit" >&3
|
||||||
|
exec 3>&-
|
||||||
|
|
||||||
|
# Wait for edge to finish
|
||||||
|
wait $EDGE_PID || true
|
||||||
|
|
||||||
|
echo "=== EDGE LOG ==="
|
||||||
|
cat "$EDGE_OUT"
|
||||||
|
echo "=== NODE LOG ==="
|
||||||
|
cat "$NODE_OUT"
|
||||||
|
|
||||||
|
# ---- Fail-fast verification of the direct two-process transcript ----
|
||||||
|
# Every branch below exits non-zero on a missing/extra token, a Node-vs-Edge
|
||||||
|
# payload mismatch, a mis-ordered or duplicated terminal event, a missing
|
||||||
|
# reconnect, or a missing command response. There is no silent skip: the previous
|
||||||
|
# version matched run_id on the [node0-msg] lines (which carry none), so its whole
|
||||||
|
# loop `continue`d and it printed PASS having compared nothing.
|
||||||
|
echo "[diagnostic] Verifying payload sequence, terminal ordering, and command responses..."
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
printf '[diagnostic] VALIDATION FAILED: %b\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Runs are dispatched in this order: two before the reconnect, one after. Each run
|
||||||
|
# streams exactly the extracted token then that token with a _TAIL suffix.
|
||||||
|
EXPECTED_TOKENS=(IOP_E2E_HELLO_BASIC IOP_E2E_HELLO_FORMAL IOP_E2E_PING_BASIC)
|
||||||
|
|
||||||
|
# Edge-observed run ids, in order, from the authoritative start events.
|
||||||
|
mapfile -t RUN_IDS < <(grep -oE '\[node0-evt\] start run_id=[A-Za-z0-9_-]+' "$EDGE_OUT" | sed 's/.*run_id=//')
|
||||||
|
if [ "${#RUN_IDS[@]}" -ne "${#EXPECTED_TOKENS[@]}" ]; then
|
||||||
|
fail "expected ${#EXPECTED_TOKENS[@]} runs, edge start events show ${#RUN_IDS[@]}: ${RUN_IDS[*]:-<none>}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Reconnect must have occurred: at least the initial + one reconnect registration.
|
||||||
|
CONNECTED_COUNT=$(grep -c 'connected reason="registered"' "$EDGE_OUT" || true)
|
||||||
|
if [ "${CONNECTED_COUNT:-0}" -lt 2 ]; then
|
||||||
|
fail "expected >=2 connected(registered) events (initial + reconnect), got ${CONNECTED_COUNT:-0}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for i in "${!RUN_IDS[@]}"; do
|
||||||
|
rid="${RUN_IDS[$i]}"
|
||||||
|
token="${EXPECTED_TOKENS[$i]}"
|
||||||
|
echo "[diagnostic] Checking run $((i + 1)) run_id=$rid token=$token"
|
||||||
|
|
||||||
|
start_ln=$(grep -nE "\[node0-evt\] start run_id=${rid}\$" "$EDGE_OUT" | head -n1 | cut -d: -f1 || true)
|
||||||
|
[ -n "$start_ln" ] || fail "run $rid: missing edge start event"
|
||||||
|
|
||||||
|
# The terminal event must appear exactly once for this run.
|
||||||
|
complete_ln=$(grep -nE "\[node0-evt\] complete run_id=${rid}( |\$)" "$EDGE_OUT" | cut -d: -f1 || true)
|
||||||
|
complete_count=$(printf '%s\n' "$complete_ln" | grep -c . || true)
|
||||||
|
[ "$complete_count" -eq 1 ] || fail "run $rid: expected exactly one complete event, got $complete_count"
|
||||||
|
[ "$complete_ln" -gt "$start_ln" ] || fail "run $rid: complete event precedes start"
|
||||||
|
|
||||||
|
# Payload tokens strictly between this run's start and complete, in order.
|
||||||
|
run_payloads=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" \
|
||||||
|
| grep -oE '\[node0-msg\] IOP_E2E_[A-Z0-9_]+' | sed 's/.*\] //' || true)
|
||||||
|
want_payloads=$(printf '%s\n%s_TAIL' "$token" "$token")
|
||||||
|
if [ "$run_payloads" != "$want_payloads" ]; then
|
||||||
|
fail "run $rid ($token): edge payload sequence mismatch\n got: [$run_payloads]\n want: [$want_payloads]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The terminal event must come strictly after the last payload line.
|
||||||
|
last_msg_rel=$(sed -n "${start_ln},${complete_ln}p" "$EDGE_OUT" | grep -nE '\[node0-msg\]' | tail -n1 | cut -d: -f1 || true)
|
||||||
|
[ -n "$last_msg_rel" ] || fail "run $rid: no payload lines in trace"
|
||||||
|
last_msg_abs=$((start_ln + last_msg_rel - 1))
|
||||||
|
[ "$complete_ln" -gt "$last_msg_abs" ] || fail "run $rid: complete event is not after the last payload"
|
||||||
|
|
||||||
|
# The Node process logs the same run with [node-event] start/complete bounds.
|
||||||
|
# Capture both the prefixed first payload line and any continuation lines
|
||||||
|
# emitted by a multiline adapter delta, then compare the sequence directly
|
||||||
|
# with the Edge relay for this exact run id.
|
||||||
|
node_start_ln=$(grep -nE "^\[node-event\] start run_id=${rid}\$" "$NODE_OUT" | cut -d: -f1 || true)
|
||||||
|
node_start_count=$(printf '%s\n' "$node_start_ln" | grep -c . || true)
|
||||||
|
[ "$node_start_count" -eq 1 ] || fail "run $rid: expected one Node start event, got $node_start_count"
|
||||||
|
node_complete_ln=$(grep -nE "^\[node-event\] complete run_id=${rid}( |\$)" "$NODE_OUT" | cut -d: -f1 || true)
|
||||||
|
node_complete_count=$(printf '%s\n' "$node_complete_ln" | grep -c . || true)
|
||||||
|
[ "$node_complete_count" -eq 1 ] || fail "run $rid: expected one Node complete event, got $node_complete_count"
|
||||||
|
[ "$node_complete_ln" -gt "$node_start_ln" ] || fail "run $rid: Node complete event precedes start"
|
||||||
|
|
||||||
|
node_payloads=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \
|
||||||
|
| sed -nE '/^\[node-message\] IOP_E2E_[A-Z0-9_]+$/{s/^\[node-message\] //;p;b}; /^IOP_E2E_[A-Z0-9_]+$/p' || true)
|
||||||
|
if [ "${IOP_DEV_RECONNECT_INJECT_FAIL:-}" != "" ] && [ "$i" -eq "$((${#RUN_IDS[@]} - 1))" ]; then
|
||||||
|
# Test-only fault injection: drop the final Node payload from the last
|
||||||
|
# run so the actual per-run Node-vs-Edge comparison must fail-fast.
|
||||||
|
node_payloads=$(printf '%s\n' "$node_payloads" | sed '$d')
|
||||||
|
fi
|
||||||
|
if [ "$node_payloads" != "$run_payloads" ]; then
|
||||||
|
fail "run $rid: Node-vs-Edge payload sequence mismatch\n node: [$node_payloads]\n edge: [$run_payloads]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
node_last_msg_rel=$(sed -n "${node_start_ln},${node_complete_ln}p" "$NODE_OUT" \
|
||||||
|
| grep -nE '^\[node-message\] |^IOP_E2E_' | tail -n1 | cut -d: -f1 || true)
|
||||||
|
[ -n "$node_last_msg_rel" ] || fail "run $rid: no Node payload lines in trace"
|
||||||
|
node_last_msg_abs=$((node_start_ln + node_last_msg_rel - 1))
|
||||||
|
[ "$node_complete_ln" -gt "$node_last_msg_abs" ] || fail "run $rid: Node complete event is not after the last payload"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Required command responses in the edge transcript (secret-safe: fixed markers only).
|
||||||
|
require_cmd() {
|
||||||
|
grep -qE "$2" "$EDGE_OUT" || fail "missing $1 command response"
|
||||||
|
}
|
||||||
|
require_cmd "/nodes" '^ node0 = test-node'
|
||||||
|
require_cmd "/capabilities" '\[node0-capabilities\]'
|
||||||
|
require_cmd "/transport" '\[node0-transport\]'
|
||||||
|
require_cmd "/sessions" '\[node0-sessions\]'
|
||||||
|
require_cmd "/terminate-session" 'terminated session .* node=node0'
|
||||||
|
|
||||||
|
echo "[diagnostic] PASS: ${#RUN_IDS[@]} runs verified — payload sequence, one terminal after the last payload, Node==Edge; reconnect observed; all five command responses present."
|
||||||
Loading…
Reference in a new issue