package wire import ( "context" "fmt" "testing" "time" "go.uber.org/zap/zaptest" proto_socket "git.toki-labs.com/toki/proto-socket/go" iop "iop/proto/gen/iop" ) func startEdgeServer(t *testing.T) (*EdgeServer, int) { t.Helper() logger := zaptest.NewLogger(t) port := getFreePort(t) listenAddr := fmt.Sprintf("127.0.0.1:%d", port) server, err := NewEdgeServer(listenAddr, logger) if err != nil { t.Fatalf("NewEdgeServer failed: %v", err) } ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) if err := server.Start(ctx); err != nil { t.Fatalf("edge server Start failed: %v", err) } t.Cleanup(func() { _ = server.Stop() }) // Give a tiny moment for the listener to bind. time.Sleep(50 * time.Millisecond) return server, port } func dialEdge(t *testing.T, ctx context.Context, port int) *proto_socket.TcpClient { t.Helper() client, err := proto_socket.DialTcp( ctx, "127.0.0.1", port, proto_socket.DefaultHeartbeatIntervalSec, proto_socket.DefaultHeartbeatWaitSec, EdgeParserMap(), ) if err != nil { t.Fatalf("failed to DialTcp: %v", err) } return client } func TestEdgeServerHelloRegistersEdge(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := dialEdge(t, ctx, port) defer client.Close() req := &iop.EdgeHelloRequest{ EdgeId: "edge-dgx-group", EdgeName: "DGX Group", Version: "1.2.3", Capabilities: []string{"node-registry", "run-dispatch"}, } res, err := proto_socket.SendRequestTyped[*iop.EdgeHelloRequest, *iop.EdgeHelloResponse]( &client.Communicator, req, 2*time.Second, ) if err != nil { t.Fatalf("SendRequestTyped failed: %v", err) } if !res.Accepted { t.Errorf("expected response.Accepted to be true, got false (reason=%q)", res.Reason) } if res.Protocol != Protocol { t.Errorf("expected response.Protocol to be %q, got %q", Protocol, res.Protocol) } if res.ServerTimeUnixNano <= 0 { t.Errorf("expected server time unix nano to be positive, got %d", res.ServerTimeUnixNano) } state, ok := server.Registry().Snapshot("edge-dgx-group") if !ok { t.Fatalf("edge not registered after hello") } if !state.Connected { t.Errorf("expected registry Connected to be true") } if state.EdgeName != "DGX Group" { t.Errorf("edge_name: got %q want %q", state.EdgeName, "DGX Group") } if len(state.Capabilities) != 2 { t.Errorf("capabilities: got %v", state.Capabilities) } if state.LastSeen.IsZero() { t.Errorf("expected LastSeen to be set") } } func TestEdgeServerRejectsMissingEdgeID(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := dialEdge(t, ctx, port) defer client.Close() req := &iop.EdgeHelloRequest{ EdgeName: "no id edge", } res, err := proto_socket.SendRequestTyped[*iop.EdgeHelloRequest, *iop.EdgeHelloResponse]( &client.Communicator, req, 2*time.Second, ) if err != nil { t.Fatalf("SendRequestTyped failed: %v", err) } if res.Accepted { t.Errorf("expected response.Accepted to be false for missing edge_id") } if res.Reason == "" { t.Errorf("expected a rejection reason for missing edge_id") } if server.Registry().Len() != 0 { t.Errorf("expected registry to stay empty, got %d entries", server.Registry().Len()) } } func TestEdgeServerDisconnectMarksEdgeDisconnected(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := dialEdge(t, ctx, port) req := &iop.EdgeHelloRequest{ EdgeId: "edge-dgx-group", EdgeName: "DGX Group", } if _, err := proto_socket.SendRequestTyped[*iop.EdgeHelloRequest, *iop.EdgeHelloResponse]( &client.Communicator, req, 2*time.Second, ); err != nil { t.Fatalf("SendRequestTyped failed: %v", err) } if err := client.Close(); err != nil { t.Fatalf("client Close failed: %v", err) } // The server detects the closed connection asynchronously; poll until the // disconnect listener flips the registry state. deadline := time.Now().Add(2 * time.Second) for { state, ok := server.Registry().Snapshot("edge-dgx-group") if ok && !state.Connected { if state.DisconnectInfo.Reason == "" { t.Errorf("expected a disconnect reason to be recorded") } return } if time.Now().After(deadline) { t.Fatalf("edge was not marked disconnected in time (ok=%v)", ok) } time.Sleep(20 * time.Millisecond) } } func sendEdgeHello(t *testing.T, client *proto_socket.TcpClient, edgeID string) { t.Helper() if _, err := proto_socket.SendRequestTyped[*iop.EdgeHelloRequest, *iop.EdgeHelloResponse]( &client.Communicator, &iop.EdgeHelloRequest{EdgeId: edgeID, EdgeName: "DGX Group"}, 2*time.Second, ); err != nil { t.Fatalf("SendRequestTyped failed: %v", err) } } func TestEdgeServerStaleDisconnectDoesNotClearReconnect(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() const edgeID = "edge-dgx-group" // First connection enrolls the edge. conn1 := dialEdge(t, ctx, port) sendEdgeHello(t, conn1, edgeID) // Second connection with the same edge_id reconnects and becomes current. conn2 := dialEdge(t, ctx, port) defer conn2.Close() sendEdgeHello(t, conn2, edgeID) // The stale first connection closing must not clear the live reconnection. if err := conn1.Close(); err != nil { t.Fatalf("conn1 Close failed: %v", err) } // Give the server time to process the stale disconnect, then assert the edge // stays connected under the second connection's token. time.Sleep(300 * time.Millisecond) state, ok := server.Registry().Snapshot(edgeID) if !ok { t.Fatalf("edge missing from registry") } if !state.Connected { t.Fatalf("stale disconnect cleared the live reconnection; Connected=false") } } func TestEdgeServerStaleConnectionNodeEventDoesNotPolluteReconnect(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() const edgeID = "edge-dgx-group" conn1 := dialEdge(t, ctx, port) defer conn1.Close() sendEdgeHello(t, conn1, edgeID) conn2 := dialEdge(t, ctx, port) defer conn2.Close() sendEdgeHello(t, conn2, edgeID) if err := conn1.Send(&iop.EdgeNodeEvent{ EventId: "evt-stale", Type: "node.connected", NodeId: "node-stale", }); err != nil { t.Fatalf("send stale EdgeNodeEvent: %v", err) } time.Sleep(300 * time.Millisecond) if got := server.Registry().NodeEvents(edgeID, ""); len(got) != 0 { t.Fatalf("stale connection event polluted registry: %+v", got) } if err := conn2.Send(&iop.EdgeNodeEvent{ EventId: "evt-current", Type: "node.connected", NodeId: "node-current", }); err != nil { t.Fatalf("send current EdgeNodeEvent: %v", err) } var records []EdgeNodeEventRecord deadline := time.Now().Add(2 * time.Second) for { records = server.Registry().NodeEvents(edgeID, "") if len(records) == 1 { break } if time.Now().After(deadline) { t.Fatalf("current node event was not recorded in time; got %d records", len(records)) } time.Sleep(20 * time.Millisecond) } if records[0].Event.GetEventId() != "evt-current" { t.Fatalf("recorded event_id: got %q want evt-current", records[0].Event.GetEventId()) } if records[0].Event.GetNodeId() != "node-current" { t.Fatalf("recorded node_id: got %q want node-current", records[0].Event.GetNodeId()) } } func TestEdgeServerRequestsStatusFromConnectedEdge(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := dialEdge(t, ctx, port) defer client.Close() const edgeID = "edge-dgx-group" // The test acts as the Edge: it answers a Control Plane status request with a // service-backed node snapshot. proto_socket.AddRequestListenerTyped(&client.Communicator, func(req *iop.EdgeStatusRequest) (*iop.EdgeStatusResponse, error) { return &iop.EdgeStatusResponse{ RequestId: req.GetRequestId(), EdgeId: edgeID, EdgeName: "DGX Group", ObservedTimeUnixNano: time.Now().UnixNano(), Nodes: []*iop.EdgeNodeSnapshot{ {NodeId: "node-1", Alias: "alpha", Label: "node0", Connected: true}, }, }, nil }) sendEdgeHello(t, client, edgeID) resp, err := server.RequestStatus(edgeID, 2*time.Second) if err != nil { t.Fatalf("RequestStatus: %v", err) } if resp.GetRequestId() == "" { t.Errorf("expected a non-empty correlation request_id") } if resp.GetEdgeId() != edgeID { t.Errorf("edge_id: got %q want %q", resp.GetEdgeId(), edgeID) } if len(resp.GetNodes()) != 1 { t.Fatalf("nodes: got %d want 1", len(resp.GetNodes())) } if node := resp.GetNodes()[0]; node.GetNodeId() != "node-1" || node.GetLabel() != "node0" { t.Errorf("node snapshot: %+v", node) } } func TestEdgeServerRecordsNodeLifecycleEventFromEdge(t *testing.T) { server, port := startEdgeServer(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := dialEdge(t, ctx, port) defer client.Close() if err := client.Send(&iop.EdgeNodeEvent{ EventId: "evt-before-hello", Type: "node.connected", NodeId: "node-early", }); err != nil { t.Fatalf("send pre-hello EdgeNodeEvent: %v", err) } time.Sleep(100 * time.Millisecond) if got := server.Registry().NodeEvents("edge-dgx-group", ""); len(got) != 0 { t.Fatalf("expected pre-hello node event to be ignored, got %+v", got) } sendEdgeHello(t, client, "edge-dgx-group") event := &iop.EdgeNodeEvent{ EventId: "evt-node-1", Type: "node.connected", Source: "edge", NodeId: "node-1", Alias: "alpha", Reason: "registered", Metadata: map[string]string{"rack": "r1"}, } if err := client.Send(event); err != nil { t.Fatalf("send EdgeNodeEvent: %v", err) } var records []EdgeNodeEventRecord deadline := time.Now().Add(2 * time.Second) for { records = server.Registry().NodeEvents("edge-dgx-group", "") if len(records) == 1 { break } if time.Now().After(deadline) { t.Fatalf("node event was not recorded in time; got %d records", len(records)) } time.Sleep(20 * time.Millisecond) } got := records[0] if got.EdgeID != "edge-dgx-group" { t.Errorf("edge_id: got %q want %q", got.EdgeID, "edge-dgx-group") } if got.Event.GetEventId() != "evt-node-1" || got.Event.GetType() != "node.connected" { t.Fatalf("unexpected recorded event: %+v", got.Event) } if got.Event.GetNodeId() != "node-1" || got.Event.GetAlias() != "alpha" { t.Fatalf("unexpected node identity: %+v", got.Event) } if got.Event.GetMetadata()["rack"] != "r1" { t.Fatalf("metadata rack: got %q", got.Event.GetMetadata()["rack"]) } if got.Event.GetTimestamp() == 0 { t.Fatal("expected registry to fill missing event timestamp") } if event.GetTimestamp() != 0 { t.Fatal("registry mutated original event timestamp") } records[0].Event.Metadata["rack"] = "mutated" again := server.Registry().NodeEvents("edge-dgx-group", "node-1") if len(again) != 1 { t.Fatalf("filtered node events: got %d want 1", len(again)) } if again[0].Event.GetMetadata()["rack"] != "r1" { t.Fatalf("registry event was mutated through returned snapshot: %+v", again[0].Event.GetMetadata()) } } func TestEdgeServerRequestStatusFailsWhenEdgeNotConnected(t *testing.T) { server, _ := startEdgeServer(t) if _, err := server.RequestStatus("missing-edge", time.Second); err == nil { t.Fatal("expected error requesting status from a non-connected edge") } } func TestEdgeRegistrySnapshotsAreSortedAndCopied(t *testing.T) { registry := NewEdgeRegistry() at := time.Unix(1780142300, 0).UTC() registry.MarkConnected(&iop.EdgeHelloRequest{ EdgeId: "edge-b", EdgeName: "Edge B", Capabilities: []string{"run"}, }, at) registry.MarkConnected(&iop.EdgeHelloRequest{ EdgeId: "edge-a", EdgeName: "Edge A", Capabilities: []string{"status"}, }, at.Add(time.Second)) snapshots := registry.Snapshots() if len(snapshots) != 2 { t.Fatalf("expected 2 snapshots, got %d", len(snapshots)) } if snapshots[0].EdgeID != "edge-a" || snapshots[1].EdgeID != "edge-b" { t.Fatalf("expected snapshots sorted by edge_id, got %+v", snapshots) } snapshots[0].Capabilities[0] = "mutated" again, ok := registry.Snapshot("edge-a") if !ok { t.Fatal("edge-a missing from registry") } if again.Capabilities[0] != "status" { t.Fatalf("snapshot capabilities alias registry storage: %+v", again.Capabilities) } } func TestEdgeRegistryNodeEventsAreBoundedAndCopied(t *testing.T) { registry := NewEdgeRegistry() receivedAt := time.Unix(1780142400, 0).UTC() for i := 0; i < recentNodeEventCapacity+2; i++ { registry.RecordNodeEvent("edge-a", &iop.EdgeNodeEvent{ EventId: fmt.Sprintf("evt-%03d", i), Type: "node.connected", NodeId: "node-1", Metadata: map[string]string{ "index": fmt.Sprintf("%d", i), }, }, receivedAt.Add(time.Duration(i)*time.Nanosecond)) } events := registry.NodeEvents("edge-a", "") if len(events) != recentNodeEventCapacity { t.Fatalf("node events len: got %d want %d", len(events), recentNodeEventCapacity) } if events[0].Event.GetEventId() != "evt-002" { t.Fatalf("expected oldest retained event evt-002, got %q", events[0].Event.GetEventId()) } events[0].Event.Metadata["index"] = "mutated" again := registry.NodeEvents("edge-a", "") if again[0].Event.GetMetadata()["index"] == "mutated" { t.Fatal("NodeEvents returned mutable registry storage") } } func TestEdgeHeartbeatWaitExceedsInterval(t *testing.T) { if EdgeHeartbeatWaitSec <= EdgeHeartbeatIntervalSec { t.Fatalf("EdgeHeartbeatWaitSec (%d) must exceed EdgeHeartbeatIntervalSec (%d)", EdgeHeartbeatWaitSec, EdgeHeartbeatIntervalSec) } }