iop/apps/edge/internal/transport/server.go

package transport

import (
	"context"
	"net"
	"strconv"
	"sync"
	"sync/atomic"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"go.uber.org/zap"
	"google.golang.org/protobuf/proto"

	edgenode "iop/apps/edge/internal/node"
	"iop/packages/go/config"
	"iop/packages/go/events"
	iop "iop/proto/gen/iop"
)

const (
	heartbeatIntervalSec = 30
	// heartbeatWaitSec mirrors the node side. See the comment in
	// apps/node/internal/transport/client.go for rationale.
	heartbeatWaitSec = 45
)

func edgeParserMap() toki.ParserMap {
	return toki.ParserMap{
		toki.TypeNameOf(&iop.RunEvent{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunEvent{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.EdgeNodeEvent{}): func(b []byte) (proto.Message, error) {
			m := &iop.EdgeNodeEvent{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.RegisterRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterRequest{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeCommandResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeCommandResponse{}
			return m, proto.Unmarshal(b, m)
		},
	}
}

// Server wraps proto-socket TcpServer and manages node connections.
type Server struct {
	tcp         *toki.TcpServer
	listen      string
	registry    *edgenode.Registry
	nodeStore   *edgenode.NodeStore
	logger      *zap.Logger
	handlerMu   sync.RWMutex
	onRunEvent  func(*iop.RunEvent)
	onNodeEvent func(*iop.EdgeNodeEvent)
	stopping    atomic.Bool
}

func NewServer(listen string, registry *edgenode.Registry, nodeStore *edgenode.NodeStore, logger *zap.Logger) (*Server, error) {
	host, portStr, err := net.SplitHostPort(listen)
	if err != nil {
		return nil, err
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return nil, err
	}

	s := &Server{listen: listen, registry: registry, nodeStore: nodeStore, logger: logger}
	s.tcp = toki.NewTcpServer(host, port, func(conn net.Conn) *toki.TcpClient {
		return toki.NewTcpClient(conn, heartbeatIntervalSec, heartbeatWaitSec, edgeParserMap())
	})
	s.tcp.OnClientConnected = s.onNodeConnected
	return s, nil
}

func (s *Server) Start(ctx context.Context) error {
	s.stopping.Store(false)
	if err := s.tcp.Start(ctx); err != nil {
		return err
	}
	s.logger.Info("edge listening for nodes", zap.String("addr", s.listen))
	return nil
}

func (s *Server) Stop() error {
	s.stopping.Store(true)
	return s.tcp.Stop()
}

func (s *Server) SetRunEventHandler(handler func(*iop.RunEvent)) {
	s.handlerMu.Lock()
	s.onRunEvent = handler
	s.handlerMu.Unlock()
}

func (s *Server) SetNodeEventHandler(handler func(*iop.EdgeNodeEvent)) {
	s.handlerMu.Lock()
	s.onNodeEvent = handler
	s.handlerMu.Unlock()
}

func (s *Server) HasRunEventHandler() bool {
	s.handlerMu.RLock()
	defer s.handlerMu.RUnlock()
	return s.onRunEvent != nil
}

func (s *Server) HasNodeEventHandler() bool {
	s.handlerMu.RLock()
	defer s.handlerMu.RUnlock()
	return s.onNodeEvent != nil
}

func (s *Server) onNodeConnected(client *toki.TcpClient) {
	s.logger.Info("node connection established")

	toki.AddListenerTyped[*iop.RunEvent](&client.Communicator, func(e *iop.RunEvent) {
		s.logger.Debug("run event received",
			zap.String("run_id", e.GetRunId()),
			zap.String("type", e.GetType()),
		)
		s.enrichRunEvent(e)
		s.handlerMu.RLock()
		handler := s.onRunEvent
		s.handlerMu.RUnlock()
		if handler != nil {
			handler(e)
		}
	})

	toki.AddRequestListenerTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&client.Communicator,
		func(req *iop.RegisterRequest) (*iop.RegisterResponse, error) {
			rec, ok := s.nodeStore.FindByToken(req.GetToken())
			if !ok {
				s.logger.Warn("unknown token", zap.String("token_prefix", safePrefix(req.GetToken())))
				return &iop.RegisterResponse{Accepted: false, Reason: "unknown token"}, nil
			}

			if _, ok := s.registry.Get(rec.ID); ok {
				reason := "node already connected"
				if rec.AgentKind == config.AgentKindOTOAgent {
					reason = "oto agent already connected"
				}
				s.logger.Warn("duplicate registration rejected",
					zap.String("node_id", rec.ID),
					zap.String("agent_kind", rec.AgentKind),
				)
				return &iop.RegisterResponse{Accepted: false, Reason: reason}, nil
			}

			cfg, err := edgenode.BuildConfigPayload(rec)
			if err != nil {
				s.logger.Error("build config payload failed",
					zap.String("node_id", rec.ID),
					zap.Error(err),
				)
				return &iop.RegisterResponse{Accepted: false, Reason: "internal config error"}, nil
			}

			entry := &edgenode.NodeEntry{
				NodeID:    rec.ID,
				Alias:     rec.Alias,
				AgentKind: rec.AgentKind,
				Client:    client,
				Index:     rec.Index,
				HasIndex:  true,
			}
			toki.AddListenerTyped[*iop.EdgeNodeEvent](&client.Communicator, func(e *iop.EdgeNodeEvent) {
				if e.NodeId == "" {
					e.NodeId = rec.ID
				}
				if e.Alias == "" {
					e.Alias = rec.Alias
				}
				s.emitNodeEvent(e)
			})
			client.AddDisconnectListener(func(_ *toki.TcpClient) {
				transportInfo := client.DisconnectInfo()
				fields := []zap.Field{zap.String("node_id", rec.ID)}
				fields = append(fields, transportDisconnectFields(transportInfo)...)
				s.logger.Info("node unregistered", fields...)
				reason := events.ReasonTransportClosed
				if s.stopping.Load() {
					reason = events.ReasonEdgeShutdown
				}
				s.emitNodeEvent(events.NewEdgeNodeEvent(
					events.SourceEdge,
					events.TypeNodeDisconnected,
					rec.ID,
					rec.Alias,
					reason,
					transportDisconnectMetadata(transportInfo),
				))
				s.registry.Unregister(rec.ID)
			})
			s.registry.Register(entry)
			s.logger.Info("node registered",
				zap.String("node_id", rec.ID),
				zap.String("alias", rec.Alias),
			)
			s.emitNodeEvent(events.NewEdgeNodeEvent(
				events.SourceEdge,
				events.TypeNodeConnected,
				rec.ID,
				rec.Alias,
				events.ReasonRegistered,
				nil,
			))

			return &iop.RegisterResponse{
				Accepted: true,
				NodeId:   rec.ID,
				Alias:    rec.Alias,
				Config:   cfg,
			}, nil
		},
	)
}

func (s *Server) enrichRunEvent(event *iop.RunEvent) {
	if event == nil || event.GetNodeAlias() != "" || s.registry == nil {
		return
	}
	nodeID := event.GetNodeId()
	if nodeID == "" {
		return
	}
	if entry, ok := s.registry.Get(nodeID); ok && entry.Alias != "" {
		event.NodeAlias = entry.Alias
	}
}

func (s *Server) emitNodeEvent(event *iop.EdgeNodeEvent) {
	s.logger.Debug("node event received",
		zap.String("node_id", event.GetNodeId()),
		zap.String("type", event.GetType()),
		zap.String("reason", event.GetReason()),
	)
	s.handlerMu.RLock()
	handler := s.onNodeEvent
	s.handlerMu.RUnlock()
	if handler != nil {
		handler(event)
	}
}

func safePrefix(s string) string {
	if len(s) > 8 {
		return s[:8] + "..."
	}
	return s
}

func transportDisconnectMetadata(info toki.DisconnectInfo) map[string]string {
	metadata := make(map[string]string, 2)
	if info.Reason != "" {
		metadata[events.MetadataTransportCloseReason] = info.Reason
	}
	if info.Error != "" {
		metadata[events.MetadataTransportCloseError] = info.Error
	}
	if len(metadata) == 0 {
		return nil
	}
	return metadata
}

func transportDisconnectFields(info toki.DisconnectInfo) []zap.Field {
	fields := make([]zap.Field, 0, 2)
	if info.Reason != "" {
		fields = append(fields, zap.String("transport_close_reason", info.Reason))
	}
	if info.Error != "" {
		fields = append(fields, zap.String("transport_close_error", info.Error))
	}
	return fields
}