iop/apps/edge/internal/bootstrap/runtime_test.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"iop/apps/edge/internal/configrefresh"
	"iop/packages/go/config"
)

func newTestConfig() *config.EdgeConfig {
	return &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Metrics: config.MetricsConf{Port: 0},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "alias-1", Token: "tok-1"},
		},
	}
}

func TestNewRuntimeSeedsNodeStore(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.NodeStore == nil {
		t.Fatal("NodeStore is nil")
	}
	rec, ok := rt.NodeStore.FindByToken("tok-1")
	if !ok {
		t.Fatal("expected node record seeded by token tok-1")
	}
	if rec.ID != "node-1" || rec.Alias != "alias-1" {
		t.Errorf("unexpected record: id=%q alias=%q", rec.ID, rec.Alias)
	}
	if rt.Registry == nil || rt.EventBus == nil || rt.Service == nil || rt.Server == nil || rt.Logger == nil {
		t.Error("runtime dependencies must be wired by NewRuntime")
	}
}

func TestRuntimeWiresTransportHandlers(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Server.HasRunEventHandler() || rt.Server.HasNodeEventHandler() {
		t.Fatal("handlers must not be wired before wireHandlers")
	}
	rt.wireHandlers()
	if !rt.Server.HasRunEventHandler() {
		t.Error("run event handler not wired")
	}
	if !rt.Server.HasNodeEventHandler() {
		t.Error("node event handler not wired")
	}
}

func TestNewRuntimeWiresInputManager(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Input == nil {
		t.Fatal("Input manager is nil")
	}
	if rt.Input.OpenAI == nil {
		t.Fatal("Input.OpenAI is nil")
	}
	if rt.Input.A2A == nil {
		t.Fatal("Input.A2A is nil")
	}
}

func TestRuntimeKeepsListenersAfterStartupContextCancel(t *testing.T) {
	edgeAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)
	cfg := newTestConfig()
	cfg.Server.Listen = edgeAddr
	cfg.OpenAI = config.EdgeOpenAIConf{
		Enabled: true,
		Listen:  openAIAddr,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	startupCtx, cancelStartup := context.WithCancel(context.Background())
	if err := rt.Start(startupCtx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	cancelStartup()

	conn, err := net.DialTimeout("tcp", edgeAddr, time.Second)
	if err != nil {
		t.Fatalf("edge listener closed with startup context: %v", err)
	}
	_ = conn.Close()

	client := &http.Client{Timeout: time.Second}
	resp, err := client.Get("http://" + openAIAddr + "/healthz")
	if err != nil {
		t.Fatalf("openai listener closed with startup context: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("openai health status = %d, want %d", resp.StatusCode, http.StatusOK)
	}
}

func TestNewRuntimeWritesLogsToConfiguredPath(t *testing.T) {
	dir := t.TempDir()
	logPath := filepath.Join(dir, "logs", "edge.log")

	cfg := newTestConfig()
	cfg.Logging.Level = "info"
	cfg.Logging.Path = logPath
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	rt.Logger.Info("runtime-log-entry")
	_ = rt.Logger.Sync()

	info, err := os.Stat(logPath)
	if err != nil {
		t.Fatalf("stat log path: %v", err)
	}
	if info.Size() == 0 {
		t.Fatalf("log file empty: %s", logPath)
	}
	contents, err := os.ReadFile(logPath)
	if err != nil {
		t.Fatalf("read log file: %v", err)
	}
	if !strings.Contains(string(contents), "runtime-log-entry") {
		t.Fatalf("expected runtime-log-entry in log file; contents=%q", string(contents))
	}
}

func TestNewRuntimeRejectsDuplicateToken(t *testing.T) {
	cfg := &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "a1", Token: "dup"},
			{ID: "node-2", Alias: "a2", Token: "dup"},
		},
	}
	if _, err := NewRuntime(cfg); err == nil {
		t.Fatal("expected error on duplicate token seed")
	}
}

// TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running
// Edge process can receive dry-run and apply refresh requests via the admin API
// and returns mode-specific results.
func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) {
	dir := t.TempDir()

	serverAddr := freeTCPAddr(t)

	// Write base and candidate YAMLs with explicit immutable field values so
	// that config.LoadEdge produces configs with identical defaults on both sides.
	// Only capacity differs between base and candidate.
	baseYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	candidateYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 8
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization so the current config matches
	// what iop-edge serve would hold after loadRuntimeConfig.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	// Override fields that don't need to match between runtime and candidate.
	currentCfg.Refresh.Enabled = true
	currentCfg.Refresh.Listen = "127.0.0.1:0"
	currentCfg.Logging.Level = "error"

	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminAddr := rt.RefreshAdmin.Addr()
	if adminAddr == "" {
		t.Fatal("refresh admin addr is empty after Start")
	}
	adminURL := "http://" + adminAddr + "/refresh"

	client := &http.Client{Timeout: 5 * time.Second}

	postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result {
		t.Helper()
		body, _ := json.Marshal(configrefresh.Request{
			Mode:       mode,
			ConfigPath: candidatePath,
			RequestID:  string(mode) + "-req",
		})
		resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
		if err != nil {
			t.Fatalf("POST /refresh mode=%s: %v", mode, err)
		}
		defer resp.Body.Close()
		var result configrefresh.Result
		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
			t.Fatalf("decode response: %v", err)
		}
		return result
	}

	// dry-run: process is reached, result has mode=dry_run.
	dryResult := postRefresh(t, configrefresh.ModeDryRun)
	if dryResult.Mode != configrefresh.ModeDryRun {
		t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode)
	}
	if dryResult.Status != configrefresh.StatusApplied {
		t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status)
	}

	// apply: process is reached, result has mode=apply and status=applied.
	applyResult := postRefresh(t, configrefresh.ModeApply)
	if applyResult.Mode != configrefresh.ModeApply {
		t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode)
	}
	if applyResult.Status != configrefresh.StatusApplied {
		t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status)
	}

	// verify dry-run and apply have distinct observable results: the apply
	// committed the capacity change, so the next dry-run shows no changes.
	dryResult2 := postRefresh(t, configrefresh.ModeDryRun)
	if len(dryResult2.Changes) != 0 {
		t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes)
	}
}

func serveNormalizedLogPath() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Join(filepath.Dir(exe), "logs", "edge.log")
	}
	if wd, err := os.Getwd(); err == nil {
		return filepath.Join(wd, "logs", "edge.log")
	}
	return "logs/edge.log"
}

func serveNormalizedArtifactDir(dir string) string {
	if dir == "" {
		dir = "artifacts"
	}
	if filepath.IsAbs(dir) {
		return dir
	}
	bd := binaryDirForTest()
	return filepath.Join(bd, dir)
}

func binaryDirForTest() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Dir(exe)
	}
	if wd, err := os.Getwd(); err == nil {
		return wd
	}
	return "."
}

func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	writeConfig := func(t *testing.T, name string, capacity int, displayName string) string {
		t.Helper()
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:19093"
openai:
  enabled: true
  listen: %q
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: %q
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, displayName, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig(t, "base.yaml", 2, "Qwen Base")
	candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate")

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	start := make(chan struct{})
	var wg sync.WaitGroup
	errCh := make(chan error, 3)

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 50; i++ {
			result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
				Mode:       configrefresh.ModeApply,
				ConfigPath: candidatePath,
				RequestID:  fmt.Sprintf("apply-%d", i),
			})
			if err != nil {
				errCh <- err
				return
			}
			if result.Status != configrefresh.StatusApplied {
				errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes)
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			_ = rt.Service.ListNodeSnapshots()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		url := "http://" + openAIAddr + "/v1/models"
		for i := 0; i < 100; i++ {
			resp, err := client.Get(url)
			if err != nil {
				errCh <- err
				return
			}
			resp.Body.Close()
			if resp.StatusCode != http.StatusOK {
				errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode)
				return
			}
		}
	}()

	close(start)
	wg.Wait()
	close(errCh)
	for err := range errCh {
		if err != nil {
			t.Fatalf("concurrent refresh reader failed: %v", err)
		}
	}
}

func freeTCPAddr(t *testing.T) string {
	t.Helper()
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen free tcp addr: %v", err)
	}
	addr := ln.Addr().String()
	if err := ln.Close(); err != nil {
		t.Fatalf("close free tcp addr listener: %v", err)
	}
	return addr
}

// TestNewRuntimeWiresControlPlaneConnector verifies that NewRuntime always
// wires a non-nil ControlPlane connector regardless of config.
func TestNewRuntimeWiresControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.ControlPlane == nil {
		t.Fatal("expected non-nil ControlPlane connector")
	}
}

// TestNewRuntimeWiresStatusProviderIntoConnector verifies that NewRuntime wires
// the edge service as the Control Plane connector's status provider, so status
// requests are answered from the Edge-owned node snapshot surface.
func TestNewRuntimeWiresStatusProviderIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.StatusProviderConfigured() {
		t.Fatal("expected NewRuntime to wire a status provider into the connector")
	}
}

// TestNewRuntimeWiresNodeEventBusIntoConnector verifies that node lifecycle
// events are relayed through the shared events.Bus boundary.
func TestNewRuntimeWiresNodeEventBusIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.NodeEventBusConfigured() {
		t.Fatal("expected NewRuntime to wire the node event bus into the connector")
	}
}

// TestRuntimeStartsControlPlaneConnectorWhenEnabled verifies that Start does
// not fail when a Control Plane connector is configured with a local fake
// server accepting hello. Uses a local TCP fake endpoint.
func TestRuntimeStartsControlPlaneConnectorWhenEnabled(t *testing.T) {
	// Start a minimal fake Control Plane server.
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen fake cp: %v", err)
	}
	cpAddr := ln.Addr().String()
	go func() {
		// Accept and immediately close; connector will fail hello and retry,
		// but Start() itself must succeed (connector errors are async).
		conn, err := ln.Accept()
		if err != nil {
			return
		}
		_ = conn.Close()
		_ = ln.Close()
	}()

	cfg := newTestConfig()
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:              true,
		WireAddr:             cpAddr,
		ReconnectIntervalSec: 60,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.IsEnabled() {
		t.Fatal("expected connector IsEnabled()==true")
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start with enabled connector: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
}

// TestRuntimeStopsControlPlaneConnector verifies that Stop() reaches the
// ControlPlane connector. A disabled connector is used so no real network
// activity occurs.
func TestRuntimeStopsControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	// disabled connector; Start/Stop must be no-ops on the connector side.
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:  false,
		WireAddr: "",
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
	// After Stop, the connector must be in a stopped-or-no-op state.
	// For disabled connector, CurrentState is Disconnected (never started).
	// Calling Stop again must not panic.
	rt.ControlPlane.Stop()
}