iop/apps/control-plane/cmd/control-plane/secure_delivery_integration_test.go

package main

import (
	"bytes"
	"crypto/ecdh"
	"crypto/ed25519"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"io"
	"math/big"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

const secureDeliveryE2EEnv = "IOP_SECURE_DELIVERY_E2E"

// TestSecureDeliveryThreeProcess is an opt-in, process-level proof for the
// managed credential path. The normal package suite skips it; the repository
// secure-delivery target enables it after the lightweight CP-Edge smoke.
func TestSecureDeliveryThreeProcess(t *testing.T) {
	if os.Getenv(secureDeliveryE2EEnv) != "1" {
		t.Skip("set IOP_SECURE_DELIVERY_E2E=1 to run the three-process fixture")
	}

	repoRoot := secureRepoRoot(t)
	workDir, err := os.MkdirTemp(repoRoot, ".tmp-secure-delivery-")
	if err != nil {
		t.Fatalf("create executable integration workspace: %v", err)
	}
	t.Cleanup(func() { _ = os.RemoveAll(workDir) })
	goTmp := filepath.Join(workDir, "gotmp")
	if err := os.MkdirAll(goTmp, 0o700); err != nil {
		t.Fatalf("create Go temporary directory: %v", err)
	}

	binaries := map[string]string{
		"control-plane": filepath.Join(workDir, "control-plane"),
		"edge":          filepath.Join(workDir, "edge"),
		"node":          filepath.Join(workDir, "node"),
	}
	for name, pkg := range map[string]string{
		"control-plane": "./apps/control-plane/cmd/control-plane",
		"edge":          "./apps/edge/cmd/edge",
		"node":          "./apps/node/cmd/node",
	} {
		cmd := exec.Command("go", "build", "-o", binaries[name], pkg)
		cmd.Dir = repoRoot
		cmd.Env = append(os.Environ(), "TMPDIR="+workDir, "GOTMPDIR="+goTmp)
		output, buildErr := cmd.CombinedOutput()
		if buildErr != nil {
			t.Fatalf("build %s: %v\n%s", name, buildErr, output)
		}
	}

	ca := secureNewCA(t, workDir)
	cpIdentity := ca.issue(t, workDir, "control-plane", "cp-secure", []string{"cp.internal", "cp-api.internal"})
	edgeIdentity := ca.issue(t, workDir, "edge", "edge-secure", []string{"edge.internal", "edge-api.internal"})
	nodeIdentity := ca.issue(t, workDir, "node", "node-secure", nil)

	issuerPublic, issuerPrivate, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatalf("generate issuer key: %v", err)
	}
	issuerPrivatePath := secureWriteBase64(t, workDir, "issuer.ed25519", issuerPrivate, 0o600)
	issuerPublicPath := secureWriteBase64(t, workDir, "issuer.ed25519.pub", issuerPublic, 0o644)
	recipientPrivate, err := ecdh.X25519().GenerateKey(rand.Reader)
	if err != nil {
		t.Fatalf("generate recipient key: %v", err)
	}
	recipientPrivatePath := secureWriteBase64(t, workDir, "recipient.x25519", recipientPrivate.Bytes(), 0o600)

	atRestMaterial := make([]byte, 32)
	if _, err := rand.Read(atRestMaterial); err != nil {
		t.Fatalf("generate at-rest key: %v", err)
	}
	manifestPath := filepath.Join(workDir, "credential-keys.yaml")
	secureWriteFile(t, manifestPath, []byte(fmt.Sprintf("keys:\n  - id: secure-e2e\n    version: 1\n    material: %s\n", base64.StdEncoding.EncodeToString(atRestMaterial))), 0o600)
	for i := range atRestMaterial {
		atRestMaterial[i] = 0
	}

	cpHTTPPort := secureFreePort(t)
	cpClientPort := secureFreePort(t)
	cpEdgePort := secureFreePort(t)
	edgeNodePort := secureFreePort(t)
	edgeBootstrapPort := secureFreePort(t)
	edgeOpenAIPort := secureFreePort(t)
	databasePath := filepath.Join(workDir, "credentials.db")

	const providerSecret = "secure-delivery-sentinel-7e2d8c"
	upstreamStarted := make(chan struct{}, 1)
	releaseUpstream := make(chan struct{})
	var releaseUpstreamOnce sync.Once
	releaseBlockedUpstream := func() { releaseUpstreamOnce.Do(func() { close(releaseUpstream) }) }
	defer releaseBlockedUpstream()
	var upstreamCalls atomic.Int32
	var upstreamAuthOK atomic.Bool
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/v1/chat/completions" {
			w.Header().Set("Content-Type", "application/json")
			_, _ = io.WriteString(w, `{"object":"list","data":[]}`)
			return
		}
		upstreamCalls.Add(1)
		upstreamAuthOK.Store(r.Header.Get("Authorization") == "Bearer "+providerSecret)
		select {
		case upstreamStarted <- struct{}{}:
		default:
		}
		select {
		case <-releaseUpstream:
		case <-r.Context().Done():
			return
		}
		w.Header().Set("Content-Type", "application/json")
		_, _ = io.WriteString(w, `{"id":"chatcmpl-secure","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
	}))
	defer upstream.Close()

	cpConfig := filepath.Join(workDir, "control-plane.yaml")
	secureWriteFile(t, cpConfig, []byte(fmt.Sprintf(`server:
  listen: "127.0.0.1:%d"
  wire_listen: "127.0.0.1:%d"
  edge_wire_listen: "127.0.0.1:%d"
  edge_wire_tls:
    enabled: true
    cert: %q
    key: %q
    ca: %q
    peer_role: "edge"
    peer_name: "edge-secure"
database:
  url: %q
credential_encryption:
  key_file: %q
  active_key_id: "secure-e2e"
  active_key_version: 1
credential_plane:
  enabled: true
  https:
    enabled: true
    cert: %q
    key: %q
  issuer_key_id: "issuer-secure"
  issuer_private_key: %q
  lease_ttl_seconds: 30
  lease_cache_size: 32
logging:
  level: "info"
  pretty: false
metrics:
  port: 0
`, cpHTTPPort, cpClientPort, cpEdgePort, cpIdentity.cert, cpIdentity.key, ca.cert, databasePath, manifestPath, cpIdentity.cert, cpIdentity.key, issuerPrivatePath)), 0o600)

	edgeConfig := filepath.Join(workDir, "edge.yaml")
	secureWriteFile(t, edgeConfig, []byte(fmt.Sprintf(`edge:
  id: "edge-secure"
  name: "Secure Delivery Edge"
server:
  listen: "127.0.0.1:%d"
bootstrap:
  listen: "127.0.0.1:%d"
  artifact_dir: %q
tls:
  enabled: true
  cert: %q
  key: %q
  ca: %q
  peer_role: "node"
  peer_name: "node-secure"
credential_plane:
  enabled: true
  lease_ttl_seconds: 30
  lease_cache_size: 32
control_plane:
  enabled: true
  wire_addr: "127.0.0.1:%d"
  reconnect_interval_sec: 1
  tls:
    enabled: true
    cert: %q
    key: %q
    ca: %q
    server_name: "cp.internal"
    peer_role: "control-plane"
    peer_name: "cp-secure"
refresh:
  enabled: false
long_context_threshold_tokens: 100000
provider_pool:
  max_queue: 4
  queue_timeout_ms: 3000
openai:
  enabled: true
  listen: "127.0.0.1:%d"
  provider_id: "provider-secure"
  session_id: "secure-e2e"
  timeout_sec: 15
  strict_output: true
  tls:
    enabled: true
    cert: %q
    key: %q
a2a:
  enabled: false
logging:
  level: "info"
  pretty: false
  path: %q
metrics:
  port: 0
models:
  - id: "catalog-secure"
    providers:
      provider-secure: "upstream-secure"
nodes:
  - id: "node-secure"
    alias: "Secure Node"
    token: "node-registration-token"
    providers:
      - id: "provider-secure"
        type: "openai_api"
        category: "api"
        profile: "openai"
        endpoint: %q
        models: ["upstream-secure"]
        health: "available"
        capacity: 1
    runtime:
      concurrency: 1
`, edgeNodePort, edgeBootstrapPort, filepath.Join(workDir, "artifacts"), edgeIdentity.cert, edgeIdentity.key, ca.cert, cpEdgePort, edgeIdentity.cert, edgeIdentity.key, ca.cert, edgeOpenAIPort, edgeIdentity.cert, edgeIdentity.key, filepath.Join(workDir, "edge.log"), upstream.URL)), 0o600)

	nodeConfig := filepath.Join(workDir, "node.yaml")
	secureWriteFile(t, nodeConfig, []byte(fmt.Sprintf(`transport:
  edge_addr: "127.0.0.1:%d"
  token: "node-registration-token"
  tls:
    enabled: true
    cert: %q
    key: %q
    ca: %q
    server_name: "edge.internal"
    peer_role: "edge"
    peer_name: "edge-secure"
credential_plane:
  enabled: true
  recipient_key_id: "recipient-secure"
  recipient_private_key: %q
  issuer_key_id: "issuer-secure"
  issuer_public_key: %q
  replay_cache_size: 32
reconnect:
  interval_sec: 1
  max_attempts: 30
logging:
  level: "info"
  pretty: false
metrics:
  port: 0
`, edgeNodePort, nodeIdentity.cert, nodeIdentity.key, ca.cert, recipientPrivatePath, issuerPublicPath)), 0o600)

	bootstrap := exec.Command(binaries["control-plane"], "--config", cpConfig, "principal", "bootstrap", "--alias", "secure-principal")
	bootstrap.Dir = workDir
	bootstrapOutput, err := bootstrap.CombinedOutput()
	if err != nil {
		t.Fatalf("bootstrap principal: %v\n%s", err, bootstrapOutput)
	}
	principalToken := strings.TrimSpace(string(bootstrapOutput))
	if len(principalToken) != 64 {
		t.Fatalf("bootstrap returned an invalid one-time token length: %d", len(principalToken))
	}

	cpProcess := secureStartProcess(t, workDir, "control-plane", binaries["control-plane"], "serve", "--config", cpConfig)
	defer cpProcess.stop(t)
	cpClient := secureHTTPSClient(t, ca.cert, "cp-api.internal")
	cpBaseURL := fmt.Sprintf("https://127.0.0.1:%d", cpHTTPPort)
	secureWaitHTTP(t, cpClient, cpBaseURL+"/healthz", nil, cpProcess)

	edgeProcess := secureStartProcess(t, workDir, "edge", binaries["edge"], "serve", "--config", edgeConfig)
	defer edgeProcess.stop(t)
	edgeClient := secureHTTPSClient(t, ca.cert, "edge-api.internal")
	edgeBaseURL := fmt.Sprintf("https://127.0.0.1:%d", edgeOpenAIPort)
	secureWaitHTTP(t, edgeClient, edgeBaseURL+"/healthz", nil, edgeProcess)

	nodeProcess := secureStartProcess(t, workDir, "node", binaries["node"], "serve", "--config", nodeConfig)
	defer nodeProcess.stop(t)

	slot := secureCredentialRequest(t, cpClient, http.MethodPost, cpBaseURL+"/v1/credentials/slots", principalToken, []byte(providerSecret), map[string]string{
		"Content-Type": "application/octet-stream", "IOP-Credential-Vendor": "openai", "IOP-Credential-Kind": "bearer", "IOP-Credential-Alias": "secure-slot",
	})
	slotID := secureJSONText(t, slot, "ID")
	routeBody, err := json.Marshal(map[string]string{
		"slot_id": slotID, "alias": "secure-route", "profile_id": "openai", "upstream_model": "upstream-secure", "resource_selector": "provider-secure",
	})
	if err != nil {
		t.Fatalf("encode route request: %v", err)
	}
	route := secureCredentialRequest(t, cpClient, http.MethodPost, cpBaseURL+"/v1/credentials/routes", principalToken, routeBody, map[string]string{"Content-Type": "application/json"})
	routeID := secureJSONText(t, route, "ID")
	routeRevision := secureJSONInt(t, route, "Revision")

	modelsHeaders := map[string]string{"Authorization": "Bearer " + principalToken}
	secureWaitHTTP(t, edgeClient, edgeBaseURL+"/v1/models", modelsHeaders, nodeProcess)
	secureWaitLog(t, nodeProcess, "connected to edge")

	requestBody := []byte(fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"hello"}],"stream":false}`, routeID))
	type responseResult struct {
		status int
		body   []byte
		err    error
	}
	requestDone := make(chan responseResult, 1)
	go func() {
		req, requestErr := http.NewRequest(http.MethodPost, edgeBaseURL+"/v1/chat/completions", bytes.NewReader(requestBody))
		if requestErr != nil {
			requestDone <- responseResult{err: requestErr}
			return
		}
		req.Header.Set("Authorization", "Bearer "+principalToken)
		req.Header.Set("Content-Type", "application/json")
		resp, requestErr := edgeClient.Do(req)
		if requestErr != nil {
			requestDone <- responseResult{err: requestErr}
			return
		}
		defer resp.Body.Close()
		body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
		requestDone <- responseResult{status: resp.StatusCode, body: body, err: readErr}
	}()

	select {
	case <-upstreamStarted:
	case result := <-requestDone:
		t.Fatalf("managed request terminated before the fake upstream: status=%d err=%v body=%s\n%s", result.status, result.err, result.body, secureProcessDiagnostics(cpProcess, edgeProcess, nodeProcess))
	case <-time.After(20 * time.Second):
		t.Fatalf("managed request did not reach the fake upstream\n%s", secureProcessDiagnostics(cpProcess, edgeProcess, nodeProcess))
	}
	if !upstreamAuthOK.Load() {
		t.Fatal("fake upstream did not receive the exact expected managed authorization header")
	}

	revokeURL := fmt.Sprintf("%s/v1/credentials/routes/%s/revoke", cpBaseURL, routeID)
	secureCredentialRequest(t, cpClient, http.MethodPost, revokeURL, principalToken, nil, map[string]string{"IOP-Expected-Revision": fmt.Sprintf("%d", routeRevision)})
	releaseBlockedUpstream()
	select {
	case result := <-requestDone:
		if result.err != nil || result.status != http.StatusOK || !bytes.Contains(result.body, []byte(`"content":"ok"`)) {
			t.Fatalf("already-started request did not finish: status=%d err=%v body=%s", result.status, result.err, result.body)
		}
	case <-time.After(20 * time.Second):
		t.Fatal("already-started request did not terminate after revoke")
	}

	secondReq, err := http.NewRequest(http.MethodPost, edgeBaseURL+"/v1/chat/completions", bytes.NewReader(requestBody))
	if err != nil {
		t.Fatalf("create post-revoke request: %v", err)
	}
	secondReq.Header.Set("Authorization", "Bearer "+principalToken)
	secondReq.Header.Set("Content-Type", "application/json")
	secondResp, err := edgeClient.Do(secondReq)
	if err != nil {
		t.Fatalf("post-revoke request: %v", err)
	}
	_, _ = io.Copy(io.Discard, io.LimitReader(secondResp.Body, 1<<20))
	secondResp.Body.Close()
	if secondResp.StatusCode >= 200 && secondResp.StatusCode < 300 {
		t.Fatalf("post-revoke request unexpectedly succeeded: status=%d", secondResp.StatusCode)
	}
	time.Sleep(250 * time.Millisecond)
	if got := upstreamCalls.Load(); got != 1 {
		t.Fatalf("post-revoke request reached upstream: calls=%d want=1", got)
	}

	nodeProcess.stop(t)
	edgeProcess.stop(t)
	cpProcess.stop(t)
	for _, process := range []*secureProcess{cpProcess, edgeProcess, nodeProcess} {
		logBytes, readErr := os.ReadFile(process.logPath)
		if readErr != nil {
			t.Fatalf("read %s log: %v", process.name, readErr)
		}
		if bytes.Contains(logBytes, []byte(providerSecret)) {
			t.Fatalf("provider secret appeared in %s process output", process.name)
		}
	}
	t.Log("secure delivery full cycle passed: HTTPS management and ingress, CP-Edge/Edge-Node mTLS, Node-sealed lease, exact upstream auth, in-flight completion, and post-revoke fence")
}

type secureIdentity struct{ cert, key string }

type secureCA struct {
	cert       string
	parsedCert *x509.Certificate
	privateKey ed25519.PrivateKey
}

func secureNewCA(t *testing.T, dir string) secureCA {
	t.Helper()
	publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatalf("generate CA key: %v", err)
	}
	now := time.Now().Add(-time.Minute)
	template := &x509.Certificate{
		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "IOP secure delivery test CA"},
		NotBefore: now, NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true,
		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
	}
	der, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey)
	if err != nil {
		t.Fatalf("create CA certificate: %v", err)
	}
	parsed, err := x509.ParseCertificate(der)
	if err != nil {
		t.Fatalf("parse CA certificate: %v", err)
	}
	path := filepath.Join(dir, "ca.pem")
	secureWriteFile(t, path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
	return secureCA{cert: path, parsedCert: parsed, privateKey: privateKey}
}

func (ca secureCA) issue(t *testing.T, dir, role, name string, dnsNames []string) secureIdentity {
	t.Helper()
	publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatalf("generate %s identity: %v", role, err)
	}
	identityURI, err := url.Parse(fmt.Sprintf("spiffe://iop/%s/%s", role, name))
	if err != nil {
		t.Fatalf("parse workload URI: %v", err)
	}
	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120))
	if err != nil {
		t.Fatalf("generate certificate serial: %v", err)
	}
	now := time.Now().Add(-time.Minute)
	template := &x509.Certificate{
		SerialNumber: serial, Subject: pkix.Name{CommonName: name}, DNSNames: dnsNames, URIs: []*url.URL{identityURI},
		NotBefore: now, NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature,
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
	}
	der, err := x509.CreateCertificate(rand.Reader, template, ca.parsedCert, publicKey, ca.privateKey)
	if err != nil {
		t.Fatalf("create %s identity: %v", role, err)
	}
	certPath := filepath.Join(dir, role+".pem")
	keyPath := filepath.Join(dir, role+".key")
	secureWriteFile(t, certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644)
	privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
	if err != nil {
		t.Fatalf("marshal %s identity key: %v", role, err)
	}
	secureWriteFile(t, keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER}), 0o600)
	return secureIdentity{cert: certPath, key: keyPath}
}

type secureProcess struct {
	name    string
	cmd     *exec.Cmd
	done    chan error
	logPath string
	logFile *os.File
	stopped atomic.Bool
}

func secureStartProcess(t *testing.T, dir, name, binary string, args ...string) *secureProcess {
	t.Helper()
	logPath := filepath.Join(dir, name+".out")
	logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
	if err != nil {
		t.Fatalf("open %s log: %v", name, err)
	}
	cmd := exec.Command(binary, args...)
	cmd.Dir = dir
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	if err := cmd.Start(); err != nil {
		logFile.Close()
		t.Fatalf("start %s: %v", name, err)
	}
	process := &secureProcess{name: name, cmd: cmd, done: make(chan error, 1), logPath: logPath, logFile: logFile}
	go func() { process.done <- cmd.Wait() }()
	return process
}

func (p *secureProcess) stop(t *testing.T) {
	t.Helper()
	if p == nil || !p.stopped.CompareAndSwap(false, true) {
		return
	}
	if p.cmd.Process != nil {
		_ = p.cmd.Process.Signal(os.Interrupt)
	}
	select {
	case <-p.done:
	case <-time.After(5 * time.Second):
		if p.cmd.Process != nil {
			_ = p.cmd.Process.Kill()
		}
		<-p.done
	}
	if err := p.logFile.Close(); err != nil {
		t.Errorf("close %s log: %v", p.name, err)
	}
}

func secureRepoRoot(t *testing.T) string {
	t.Helper()
	workingDir, err := os.Getwd()
	if err != nil {
		t.Fatalf("get working directory: %v", err)
	}
	root, err := filepath.Abs(filepath.Join(workingDir, "../../../.."))
	if err != nil {
		t.Fatalf("resolve repository root: %v", err)
	}
	if _, err := os.Stat(filepath.Join(root, "go.mod")); err != nil {
		t.Fatalf("repository root is invalid: %v", err)
	}
	return root
}

func secureFreePort(t *testing.T) int {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("allocate port: %v", err)
	}
	defer listener.Close()
	return listener.Addr().(*net.TCPAddr).Port
}

func secureHTTPSClient(t *testing.T, caPath, serverName string) *http.Client {
	t.Helper()
	caPEM, err := os.ReadFile(caPath)
	if err != nil {
		t.Fatalf("read CA: %v", err)
	}
	roots := x509.NewCertPool()
	if !roots.AppendCertsFromPEM(caPEM) {
		t.Fatal("parse CA certificate")
	}
	return &http.Client{Timeout: 20 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{
		MinVersion: tls.VersionTLS13, RootCAs: roots, ServerName: serverName,
	}}}
}

func secureWaitHTTP(t *testing.T, client *http.Client, endpoint string, headers map[string]string, process *secureProcess) {
	t.Helper()
	deadline := time.Now().Add(30 * time.Second)
	for time.Now().Before(deadline) {
		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
		if err != nil {
			t.Fatalf("create readiness request: %v", err)
		}
		for name, value := range headers {
			req.Header.Set(name, value)
		}
		resp, err := client.Do(req)
		if err == nil {
			_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
			resp.Body.Close()
			if resp.StatusCode == http.StatusOK {
				return
			}
		}
		select {
		case processErr := <-process.done:
			process.stopped.Store(true)
			_ = process.logFile.Close()
			logBytes, _ := os.ReadFile(process.logPath)
			t.Fatalf("%s exited before readiness: %v\n%s", process.name, processErr, logBytes)
		default:
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("readiness timeout for %s\n%s", endpoint, secureProcessDiagnostics(process))
}

func secureWaitLog(t *testing.T, process *secureProcess, marker string) {
	t.Helper()
	deadline := time.Now().Add(30 * time.Second)
	for time.Now().Before(deadline) {
		logBytes, _ := os.ReadFile(process.logPath)
		if bytes.Contains(logBytes, []byte(marker)) {
			return
		}
		select {
		case processErr := <-process.done:
			process.stopped.Store(true)
			_ = process.logFile.Close()
			t.Fatalf("%s exited while waiting for %q: %v\n%s", process.name, marker, processErr, logBytes)
		default:
		}
		time.Sleep(100 * time.Millisecond)
	}
	t.Fatalf("timeout waiting for %q\n%s", marker, secureProcessDiagnostics(process))
}

func secureCredentialRequest(t *testing.T, client *http.Client, method, endpoint, token string, body []byte, headers map[string]string) map[string]any {
	t.Helper()
	req, err := http.NewRequest(method, endpoint, bytes.NewReader(body))
	if err != nil {
		t.Fatalf("create credential request: %v", err)
	}
	req.Header.Set("Authorization", "Bearer "+token)
	for name, value := range headers {
		req.Header.Set(name, value)
	}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatalf("credential request: %v", err)
	}
	defer resp.Body.Close()
	responseBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		t.Fatalf("read credential response: %v", err)
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		t.Fatalf("credential request failed: method=%s status=%d body=%s", method, resp.StatusCode, responseBody)
	}
	var decoded map[string]any
	if err := json.Unmarshal(responseBody, &decoded); err != nil {
		t.Fatalf("decode credential response: %v body=%s", err, responseBody)
	}
	return decoded
}

func secureJSONText(t *testing.T, value map[string]any, field string) string {
	t.Helper()
	textValue, ok := value[field].(string)
	if !ok || strings.TrimSpace(textValue) == "" {
		t.Fatalf("credential response field %s is missing", field)
	}
	return textValue
}

func secureJSONInt(t *testing.T, value map[string]any, field string) int64 {
	t.Helper()
	number, ok := value[field].(float64)
	if !ok || number < 0 {
		t.Fatalf("credential response field %s is invalid", field)
	}
	return int64(number)
}

func secureWriteBase64(t *testing.T, dir, name string, value []byte, mode os.FileMode) string {
	t.Helper()
	path := filepath.Join(dir, name)
	secureWriteFile(t, path, []byte(base64.StdEncoding.EncodeToString(value)+"\n"), mode)
	return path
}

func secureWriteFile(t *testing.T, path string, value []byte, mode os.FileMode) {
	t.Helper()
	if err := os.WriteFile(path, value, mode); err != nil {
		t.Fatalf("write %s: %v", filepath.Base(path), err)
	}
}

func secureProcessDiagnostics(processes ...*secureProcess) string {
	var diagnostics strings.Builder
	for _, process := range processes {
		if process == nil {
			continue
		}
		logBytes, _ := os.ReadFile(process.logPath)
		fmt.Fprintf(&diagnostics, "=== %s ===\n%s\n", process.name, logBytes)
	}
	return diagnostics.String()
}