iop/apps/edge/internal/transport/server.go

package transport

import (
	"context"
	"net"
	"strconv"
	"sync"
	"sync/atomic"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"git.toki-labs.com/toki/proto-socket/go/packets"
	"go.uber.org/zap"
	"google.golang.org/protobuf/proto"

	edgenode "iop/apps/edge/internal/node"
	"iop/packages/go/config"
	"iop/packages/go/events"
	iop "iop/proto/gen/iop"
)

const (
	heartbeatIntervalSec = 30
	// heartbeatWaitSec mirrors the node side. See the comment in
	// apps/node/internal/transport/client.go for rationale.
	heartbeatWaitSec = 45
)

func edgeParserMap() toki.ParserMap {
	return toki.ParserMap{
		toki.TypeNameOf(&iop.RunEvent{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunEvent{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.EdgeNodeEvent{}): func(b []byte) (proto.Message, error) {
			m := &iop.EdgeNodeEvent{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.RegisterRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterRequest{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeCommandResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeCommandResponse{}
			return m, proto.Unmarshal(b, m)
		},
	}
}

// Server wraps proto-socket TcpServer and manages node connections.
type Server struct {
	tcp               *toki.TcpServer
	listen            string
	registry          *edgenode.Registry
	nodeStore         *edgenode.NodeStore
	logger            *zap.Logger
	handlerMu         sync.RWMutex
	onRunEvent        func(*iop.RunEvent)
	onNodeEvent       func(*iop.EdgeNodeEvent)
	stopping          atomic.Bool
	HeartbeatInterval int
	HeartbeatWait     int
}

func NewServer(listen string, registry *edgenode.Registry, nodeStore *edgenode.NodeStore, logger *zap.Logger) (*Server, error) {
	host, portStr, err := net.SplitHostPort(listen)
	if err != nil {
		return nil, err
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return nil, err
	}

	s := &Server{
		listen:            listen,
		registry:          registry,
		nodeStore:         nodeStore,
		logger:            logger,
		HeartbeatInterval: heartbeatIntervalSec,
		HeartbeatWait:     heartbeatWaitSec,
	}
	s.tcp = toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
		return toki.NewTcpClient(conn, s.HeartbeatInterval, s.HeartbeatWait, edgeParserMap())
	})
	s.tcp.OnClientConnected = s.onNodeConnected
	return s, nil
}

func (s *Server) Start(ctx context.Context) error {
	s.stopping.Store(false)
	if err := s.tcp.Start(ctx); err != nil {
		return err
	}
	s.logger.Info("edge listening for nodes", zap.String("addr", s.listen))
	return nil
}

func (s *Server) Stop() error {
	s.stopping.Store(true)
	return s.tcp.Stop()
}

func (s *Server) SetRunEventHandler(handler func(*iop.RunEvent)) {
	s.handlerMu.Lock()
	s.onRunEvent = handler
	s.handlerMu.Unlock()
}

func (s *Server) SetNodeEventHandler(handler func(*iop.EdgeNodeEvent)) {
	s.handlerMu.Lock()
	s.onNodeEvent = handler
	s.handlerMu.Unlock()
}

func (s *Server) HasRunEventHandler() bool {
	s.handlerMu.RLock()
	defer s.handlerMu.RUnlock()
	return s.onRunEvent != nil
}

func (s *Server) HasNodeEventHandler() bool {
	s.handlerMu.RLock()
	defer s.handlerMu.RUnlock()
	return s.onNodeEvent != nil
}

func (s *Server) onNodeConnected(client *toki.TcpClient) {
	s.logger.Info("node connection established")

	toki.AddListenerTyped[*iop.RunEvent](&client.Communicator, func(e *iop.RunEvent) {
		s.logger.Debug("run event received",
			zap.String("run_id", e.GetRunId()),
			zap.String("type", e.GetType()),
		)
		s.enrichRunEvent(e)
		s.handlerMu.RLock()
		handler := s.onRunEvent
		s.handlerMu.RUnlock()
		if handler != nil {
			handler(e)
		}
	})

	toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&client.Communicator,
		func(req *iop.RegisterRequest) (*iop.RegisterResponse, error) {
			rec, ok := s.nodeStore.FindByToken(req.GetToken())
			if !ok {
				s.logger.Warn("unknown token", zap.String("token_prefix", safePrefix(req.GetToken())))
				s.emitNodeEvent(events.NewEdgeNodeEvent(
					events.SourceEdge,
					events.TypeNodeRegistrationFailed,
					"",
					"",
					events.ReasonUnknownToken,
					map[string]string{
						events.MetadataFailureReason: events.ReasonUnknownToken,
						events.MetadataTokenPrefix:   safePrefix(req.GetToken()),
					},
				))
				return &iop.RegisterResponse{Accepted: false, Reason: "unknown token"}, nil
			}

			if _, ok := s.registry.Get(rec.ID); ok {
				reason := "node already connected"
				if rec.AgentKind == config.AgentKindOTOAgent {
					reason = "oto agent already connected"
				}
				s.logger.Warn("duplicate registration rejected",
					zap.String("node_id", rec.ID),
					zap.String("agent_kind", rec.AgentKind),
				)
				s.emitNodeEvent(events.NewEdgeNodeEvent(
					events.SourceEdge,
					events.TypeNodeRegistrationFailed,
					rec.ID,
					rec.Alias,
					events.ReasonDuplicateConnection,
					map[string]string{
						events.MetadataFailureReason: events.ReasonDuplicateConnection,
						events.MetadataAgentKind:     rec.AgentKind,
					},
				))
				return &iop.RegisterResponse{Accepted: false, Reason: reason}, nil
			}

			cfg, err := edgenode.BuildConfigPayload(rec)
			if err != nil {
				s.logger.Error("build config payload failed",
					zap.String("node_id", rec.ID),
					zap.Error(err),
				)
				return &iop.RegisterResponse{Accepted: false, Reason: "internal config error"}, nil
			}

			lifecycle := edgenode.LifecycleConnected
			if rec.AgentKind == config.AgentKindOTOAgent {
				lifecycle = edgenode.LifecycleAccepted
			}
			entry := &edgenode.NodeEntry{
				NodeID:         rec.ID,
				Alias:          rec.Alias,
				AgentKind:      rec.AgentKind,
				LifecycleState: lifecycle,
				Client:         client,
				Index:          rec.Index,
				HasIndex:       true,
			}
			toki.AddListenerTyped[*iop.EdgeNodeEvent](&client.Communicator, func(e *iop.EdgeNodeEvent) {
				if e.NodeId == "" {
					e.NodeId = rec.ID
				}
				if e.Alias == "" {
					e.Alias = rec.Alias
				}
				s.emitNodeEvent(e)
			})
			if rec.AgentKind == config.AgentKindOTOAgent {
				var onlineOnce sync.Once
				toki.AddListenerTyped[*packets.HeartBeat](&client.Communicator, func(*packets.HeartBeat) {
					onlineOnce.Do(func() {
						if s.registry.UpdateLifecycle(rec.ID, edgenode.LifecycleOnline) {
							s.emitNodeEvent(events.NewEdgeNodeEvent(
								events.SourceEdge,
								events.TypeNodeOnline,
								rec.ID,
								rec.Alias,
								events.ReasonFirstHeartbeat,
								map[string]string{
									events.MetadataAgentKind:      rec.AgentKind,
									events.MetadataLifecycleState: edgenode.LifecycleOnline,
								},
							))
						}
					})
				})
			}
			client.AddDisconnectListener(func(_ *toki.TcpClient) {
				transportInfo := client.DisconnectInfo()
				fields := []zap.Field{zap.String("node_id", rec.ID)}
				fields = append(fields, transportDisconnectFields(transportInfo)...)
				s.logger.Info("node unregistered", fields...)
				reason := events.ReasonTransportClosed
				if transportInfo.Reason == "heartbeat_timeout" {
					reason = events.ReasonHeartbeatTimeout
					s.registry.UpdateLifecycle(rec.ID, edgenode.LifecycleFailed)
				} else if s.stopping.Load() {
					reason = events.ReasonEdgeShutdown
				}

				meta := transportDisconnectMetadata(transportInfo)
				if meta == nil {
					meta = make(map[string]string)
				}
				if reason == events.ReasonHeartbeatTimeout {
					meta[events.MetadataFailureReason] = events.ReasonHeartbeatTimeout
					meta[events.MetadataLifecycleState] = edgenode.LifecycleFailed
				} else {
					meta[events.MetadataLifecycleState] = edgenode.LifecycleFailed
				}
				meta[events.MetadataAgentKind] = rec.AgentKind

				s.emitNodeEvent(events.NewEdgeNodeEvent(
					events.SourceEdge,
					events.TypeNodeDisconnected,
					rec.ID,
					rec.Alias,
					reason,
					meta,
				))
				s.registry.Unregister(rec.ID)
			})
			s.registry.Register(entry)
			s.logger.Info("node registered",
				zap.String("node_id", rec.ID),
				zap.String("alias", rec.Alias),
			)
			s.emitNodeEvent(events.NewEdgeNodeEvent(
				events.SourceEdge,
				events.TypeNodeConnected,
				rec.ID,
				rec.Alias,
				events.ReasonRegistered,
				map[string]string{
					events.MetadataAgentKind:      rec.AgentKind,
					events.MetadataLifecycleState: lifecycle,
				},
			))

			return &iop.RegisterResponse{
				Accepted: true,
				NodeId:   rec.ID,
				Alias:    rec.Alias,
				Config:   cfg,
			}, nil
		},
	)
}

func (s *Server) enrichRunEvent(event *iop.RunEvent) {
	if event == nil || event.GetNodeAlias() != "" || s.registry == nil {
		return
	}
	nodeID := event.GetNodeId()
	if nodeID == "" {
		return
	}
	if entry, ok := s.registry.Get(nodeID); ok && entry.Alias != "" {
		event.NodeAlias = entry.Alias
	}
}

func (s *Server) emitNodeEvent(event *iop.EdgeNodeEvent) {
	s.logger.Debug("node event received",
		zap.String("node_id", event.GetNodeId()),
		zap.String("type", event.GetType()),
		zap.String("reason", event.GetReason()),
	)
	s.handlerMu.RLock()
	handler := s.onNodeEvent
	s.handlerMu.RUnlock()
	if handler != nil {
		handler(event)
	}
}

func safePrefix(s string) string {
	if len(s) > 8 {
		return s[:8] + "..."
	}
	return s
}

func transportDisconnectMetadata(info toki.DisconnectInfo) map[string]string {
	metadata := make(map[string]string, 2)
	if info.Reason != "" {
		metadata[events.MetadataTransportCloseReason] = info.Reason
	}
	if info.Error != "" {
		metadata[events.MetadataTransportCloseError] = info.Error
	}
	if len(metadata) == 0 {
		return nil
	}
	return metadata
}

func transportDisconnectFields(info toki.DisconnectInfo) []zap.Field {
	fields := make([]zap.Field, 0, 2)
	if info.Reason != "" {
		fields = append(fields, zap.String("transport_close_reason", info.Reason))
	}
	if info.Error != "" {
		fields = append(fields, zap.String("transport_close_error", info.Error))
	}
	return fields
}