toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/scripts/e2e-credential-slot-smoke.sh
toki 4c8441e6c9 feat(credential): Provider Credential Slot 라우팅을 구현한다 
사용자별 credential 저장, lease, projection, runtime 전달과 OpenAI-compatible 계약 및 검증 근거를 함께 반영한다.
2026-08-02 09:10:11 +09:00

1142 lines
51 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
# Credential-slot qualification modes:
# no arguments deterministic two-slot full-cycle smoke
# --self-test-live-gate deterministic live gate and one-call proof
# --live ... explicitly authorized representative one-shot
MODE="deterministic"
MODE_SET=0
SECRET_STDIN=0
LIVE_PROFILE=""
LIVE_MODEL=""
LIVE_REASONING_EFFORT=""
LIVE_MAX_COMPLETION_TOKENS=""
early_die() {
printf '[credential-slot-smoke] ERROR: %s\n' "$*" >&2
exit 2
}
# Parse and validate the complete mode boundary before creating temporary
# files, reading stdin, building binaries, or opening a socket.
while (($# > 0)); do
case "$1" in
--live)
((MODE_SET == 0)) || early_die "exactly one mode may be selected"
MODE="live"
MODE_SET=1
shift
;;
--self-test-live-gate)
((MODE_SET == 0)) || early_die "exactly one mode may be selected"
MODE="self-test-live-gate"
MODE_SET=1
shift
;;
--secret-stdin)
SECRET_STDIN=1
shift
;;
--profile)
(($# >= 2)) || early_die "--profile requires a value"
LIVE_PROFILE="$2"
shift 2
;;
--model)
(($# >= 2)) || early_die "--model requires a value"
LIVE_MODEL="$2"
shift 2
;;
--reasoning-effort)
(($# >= 2)) || early_die "--reasoning-effort requires a value"
LIVE_REASONING_EFFORT="$2"
shift 2
;;
--max-completion-tokens)
(($# >= 2)) || early_die "--max-completion-tokens requires a value"
LIVE_MAX_COMPLETION_TOKENS="$2"
shift 2
;;
*)
early_die "unknown argument: $1"
;;
esac
done
if [ "$MODE" = "live" ]; then
[ "${IOP_ALLOW_LIVE_PROVIDER:-0}" = "1" ] || early_die "live mode requires IOP_ALLOW_LIVE_PROVIDER=1"
[ "$SECRET_STDIN" = "1" ] || early_die "live mode requires --secret-stdin"
[ -n "$LIVE_PROFILE" ] || early_die "live mode requires --profile"
[ -n "$LIVE_MODEL" ] || early_die "live mode requires --model"
case "$LIVE_REASONING_EFFORT" in
minimal|low|medium|high|xhigh) ;;
*) early_die "live mode requires a valid --reasoning-effort" ;;
esac
[[ "$LIVE_MAX_COMPLETION_TOKENS" =~ ^[1-9][0-9]*$ ]] || early_die "live mode requires a positive --max-completion-tokens"
((LIVE_MAX_COMPLETION_TOKENS <= 128)) || early_die "live mode max completion tokens must not exceed 128"
elif [ "$SECRET_STDIN" = "1" ] || [ -n "$LIVE_PROFILE$LIVE_MODEL$LIVE_REASONING_EFFORT$LIVE_MAX_COMPLETION_TOKENS" ]; then
early_die "live options require --live"
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
FIXTURE="$SCRIPT_DIR/fixtures/credential-slot-vendors.json"
KEEP_TMP="${IOP_CREDENTIAL_SLOT_SMOKE_KEEP_TMP:-0}"
HTTP_CONNECT_TIMEOUT=2
HTTP_PROBE_TIMEOUT=3
HTTP_COMPLETION_TIMEOUT=45
QUIET=0
TMP_DIR=""
TEMP_PARENT=""
HELPER_BIN=""
FAKE_PID=""
CP_PID=""
EDGE_PID=""
NODE_PID=""
IOP_TOKEN=""
CHAT_SECRET=""
MESSAGES_SECRET=""
ROTATED_SECRET=""
log() {
if [ "$QUIET" != "1" ]; then
printf '[credential-slot-smoke] %s\n' "$*"
fi
}
die() {
printf '[credential-slot-smoke] ERROR: %s\n' "$*" >&2
exit 1
}
redacted_tail() {
local file="$1" line
[ -s "$file" ] || return 0
printf '=== %s (sanitized tail) ===\n' "$(basename "$file")" >&2
while IFS= read -r line; do
[ -z "$IOP_TOKEN" ] || line="${line//$IOP_TOKEN/[REDACTED_IOP_TOKEN]}"
[ -z "$CHAT_SECRET" ] || line="${line//$CHAT_SECRET/[REDACTED_PROVIDER_SECRET]}"
[ -z "$MESSAGES_SECRET" ] || line="${line//$MESSAGES_SECRET/[REDACTED_PROVIDER_SECRET]}"
[ -z "$ROTATED_SECRET" ] || line="${line//$ROTATED_SECRET/[REDACTED_PROVIDER_SECRET]}"
printf '%s\n' "$line" >&2
done < <(tail -n 60 "$file")
}
cleanup() {
local rc=$? pid file deadline
trap - EXIT INT TERM
for pid in "$NODE_PID" "$EDGE_PID" "$CP_PID" "$FAKE_PID"; do
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
kill -TERM "$pid" 2>/dev/null || true
fi
done
deadline=$((SECONDS + 5))
while ((SECONDS < deadline)); do
local alive=0
for pid in "$NODE_PID" "$EDGE_PID" "$CP_PID" "$FAKE_PID"; do
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
alive=1
fi
done
[ "$alive" = "1" ] || break
sleep 0.1
done
for pid in "$NODE_PID" "$EDGE_PID" "$CP_PID" "$FAKE_PID"; do
if [ -n "$pid" ]; then
kill -KILL "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
fi
done
if [ "$rc" -ne 0 ] && [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then
for file in "$TMP_DIR"/{fake,control-plane,edge,edge-runtime,node}.log; do
redacted_tail "$file"
done
printf '[credential-slot-smoke] failure evidence was sanitized before cleanup\n' >&2
fi
if [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then
if [ "$KEEP_TMP" = "1" ]; then
printf '[credential-slot-smoke] retained temporary evidence: %s\n' "$TMP_DIR" >&2
else
case "$TMP_DIR" in
"$TEMP_PARENT"/iop-credential-slot-smoke.*) rm -rf -- "$TMP_DIR" ;;
*) printf '[credential-slot-smoke] refused unsafe cleanup path: %s\n' "$TMP_DIR" >&2 ;;
esac
fi
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
require_tools() {
local required
for required in bash curl go jq openssl rg timeout; do
command -v "$required" >/dev/null 2>&1 || die "$required is required"
done
[ -r "$FIXTURE" ] || die "fixture is not readable: $FIXTURE"
}
setup_workspace() {
TEMP_PARENT="${TMPDIR:-/tmp}"
[ -d "$TEMP_PARENT" ] || die "TMPDIR parent does not exist: $TEMP_PARENT"
[ -w "$TEMP_PARENT" ] || die "TMPDIR parent is not writable: $TEMP_PARENT"
TMP_DIR="$(mktemp -d "$TEMP_PARENT/iop-credential-slot-smoke.XXXXXX")"
umask 077
mkdir -p "$TMP_DIR/go-tmp" "$TMP_DIR/artifacts"
printf '#!/usr/bin/env sh\nexit 0\n' >"$TMP_DIR/executable-probe"
chmod 700 "$TMP_DIR/executable-probe"
if ! "$TMP_DIR/executable-probe"; then
die "TMPDIR child is not executable; set TMPDIR to a writable executable parent"
fi
rm -f "$TMP_DIR/executable-probe"
export GOTMPDIR="$TMP_DIR/go-tmp"
}
validate_fixture() {
jq -e '
.profiles | length >= 4 and
any(.[]; .id == "seulgi_chat" and .driver == "openai_chat" and .operation == "chat_completions" and .operation_path == "/v1/chat/completions" and .auth_header == "Authorization" and .auth_scheme == "Bearer") and
any(.[]; .id == "seulgi_messages" and .driver == "anthropic_messages" and .operation == "messages" and .operation_path == "/v1/messages" and .auth_header == "x-api-key" and .auth_scheme == "")
' "$FIXTURE" >/dev/null || die "fixture profile contract is invalid"
jq -e '
.routes | length == 2 and
([.[].upstream_model] | unique | length == 1) and
([.[].slot_alias] | unique | length == 2) and
([.[].route_alias] | unique | length == 2) and
([.[].provider_id] | unique | length == 2)
' "$FIXTURE" >/dev/null || die "fixture route contract is invalid"
}
write_helper_source() {
cat >"$TMP_DIR/credential_smoke_helper.go" <<'GO'
package main
import (
"crypto/ecdh"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"database/sql"
"encoding/base64"
"encoding/json"
"encoding/pem"
"flag"
"fmt"
"io"
"math/big"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
)
type profile struct {
ID string `json:"id"`
Vendor string `json:"vendor"`
CredentialKind string `json:"credential_kind"`
Driver string `json:"driver"`
BaseURL string `json:"base_url"`
UpstreamModel string `json:"upstream_model"`
Operation string `json:"operation"`
OperationPath string `json:"operation_path"`
AuthHeader string `json:"auth_header"`
AuthScheme string `json:"auth_scheme"`
}
type fixture struct { Profiles []profile `json:"profiles"` }
func must(err error) {
if err != nil { fmt.Fprintln(os.Stderr, err); os.Exit(1) }
}
func writeFile(path string, body []byte, mode os.FileMode) {
must(os.WriteFile(path, body, mode))
}
func randomBytes(size int) []byte {
value := make([]byte, size)
_, err := io.ReadFull(rand.Reader, value)
must(err)
return value
}
type caMaterial struct { cert *x509.Certificate; key ed25519.PrivateKey; pem []byte }
func newCA(commonName string) caMaterial {
pub, key, err := ed25519.GenerateKey(rand.Reader); must(err)
now := time.Now().Add(-time.Minute)
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: commonName},
NotBefore: now, NotAfter: now.Add(2*time.Hour), IsCA: true, BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, key); must(err)
parsed, err := x509.ParseCertificate(der); must(err)
return caMaterial{cert: parsed, key: key, pem: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})}
}
func issue(dir, fileBase, role, name string, dns []string, ca caMaterial) {
pub, key, err := ed25519.GenerateKey(rand.Reader); must(err)
identity, err := url.Parse("spiffe://iop/"+role+"/"+name); must(err)
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120)); must(err)
now := time.Now().Add(-time.Minute)
tmpl := &x509.Certificate{
SerialNumber: serial, Subject: pkix.Name{CommonName: name}, DNSNames: dns, URIs: []*url.URL{identity},
NotBefore: now, NotAfter: now.Add(2*time.Hour), KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, pub, ca.key); must(err)
keyDER, err := x509.MarshalPKCS8PrivateKey(key); must(err)
writeFile(filepath.Join(dir, fileBase+".pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0644)
writeFile(filepath.Join(dir, fileBase+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE"+" KEY", Bytes: keyDER}), 0600)
}
func material(dir string) {
ca := newCA("IOP credential smoke CA")
writeFile(filepath.Join(dir, "ca.pem"), ca.pem, 0644)
other := newCA("IOP unrelated CA")
writeFile(filepath.Join(dir, "other-ca.pem"), other.pem, 0644)
issue(dir, "control-plane", "control-plane", "cp-smoke", []string{"cp.internal", "cp-api.internal"}, ca)
issue(dir, "edge", "edge", "edge-smoke", []string{"edge.internal", "edge-api.internal"}, ca)
issue(dir, "node", "node", "node-smoke", nil, ca)
issue(dir, "wrong", "worker", "wrong-smoke", nil, ca)
issuerPublic, issuerPrivate, err := ed25519.GenerateKey(rand.Reader); must(err)
writeFile(filepath.Join(dir, "issuer.private"), []byte(base64.StdEncoding.EncodeToString(issuerPrivate)+"\n"), 0600)
writeFile(filepath.Join(dir, "issuer.public"), []byte(base64.StdEncoding.EncodeToString(issuerPublic)+"\n"), 0644)
recipient, err := ecdh.X25519().GenerateKey(rand.Reader); must(err)
writeFile(filepath.Join(dir, "recipient.private"), []byte(base64.StdEncoding.EncodeToString(recipient.Bytes())+"\n"), 0600)
keyMaterial := base64.StdEncoding.EncodeToString(randomBytes(32))
writeFile(filepath.Join(dir, "keyring.yaml"), []byte("keys:\n - id: smoke-key\n version: 1\n material: "+keyMaterial+"\n"), 0600)
for _, name := range []string{"chat.secret", "messages.secret", "rotated.secret"} {
value := "cred_"+base64.RawURLEncoding.EncodeToString(randomBytes(24))
writeFile(filepath.Join(dir, name), []byte(value), 0600)
}
}
type counters struct {
mu sync.Mutex
Calls int `json:"calls"`
ChatCalls int `json:"chat_calls"`
MessagesCalls int `json:"messages_calls"`
ChatAuthOK int `json:"chat_auth_ok"`
MessagesAuthOK int `json:"messages_auth_ok"`
OptionsOK int `json:"options_ok"`
Unauthorized int `json:"unauthorized"`
}
func (c *counters) snapshot() map[string]int {
c.mu.Lock(); defer c.mu.Unlock()
return map[string]int{
"calls": c.Calls, "chat_calls": c.ChatCalls, "messages_calls": c.MessagesCalls,
"chat_auth_ok": c.ChatAuthOK, "messages_auth_ok": c.MessagesAuthOK,
"options_ok": c.OptionsOK, "unauthorized": c.Unauthorized,
}
}
func readSecret(path string) string {
value, err := os.ReadFile(path); must(err)
return strings.TrimSpace(string(value))
}
func expectedHeader(p profile, secret string) string {
if strings.TrimSpace(p.AuthScheme) == "" { return secret }
return strings.TrimSpace(p.AuthScheme)+" "+secret
}
func exactAuth(r *http.Request, p profile, secret string) bool {
if r.Header.Get(p.AuthHeader) != expectedHeader(p, secret) { return false }
if strings.EqualFold(p.AuthHeader, "Authorization") && r.Header.Get("x-api-key") != "" { return false }
if strings.EqualFold(p.AuthHeader, "x-api-key") && r.Header.Get("Authorization") != "" { return false }
return true
}
func loadFixture(path string) fixture {
data, err := os.ReadFile(path); must(err)
var value fixture
must(json.Unmarshal(data, &value))
return value
}
func findProfile(value fixture, id string) profile {
for _, candidate := range value.Profiles { if candidate.ID == id { return candidate } }
panic("profile not found: "+id)
}
func fake(args []string) {
fs := flag.NewFlagSet("fake", flag.ExitOnError)
listen := fs.String("listen", "", "listen address")
fixturePath := fs.String("fixture", "", "fixture path")
chatProfileID := fs.String("chat-profile", "", "chat profile id")
messagesProfileID := fs.String("messages-profile", "", "messages profile id")
chatSecretPath := fs.String("chat-secret", "", "chat secret path")
messagesSecretPath := fs.String("messages-secret", "", "messages secret path")
model := fs.String("model", "", "expected model")
reasoning := fs.String("reasoning-effort", "", "expected reasoning effort")
maxTokens := fs.Int("max-completion-tokens", 0, "expected completion cap")
must(fs.Parse(args))
value := loadFixture(*fixturePath)
chatProfile := findProfile(value, *chatProfileID)
var messagesProfile profile
if *messagesProfileID != "" { messagesProfile = findProfile(value, *messagesProfileID) }
state := &counters{}
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
mux.HandleFunc("/stats", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(state.snapshot())
})
mux.HandleFunc("/v1/models", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"object":"list","data":[{"id":%q}]}`, *model)
})
chatHandler := func(w http.ResponseWriter, r *http.Request) {
var request struct {
Model string `json:"model"`
Stream bool `json:"stream"`
ReasoningEffort string `json:"reasoning_effort"`
MaxCompletionTokens int `json:"max_completion_tokens"`
}
if r.Method != http.MethodPost || json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&request) != nil || request.Model != *model || request.Stream {
http.Error(w, "invalid request", http.StatusBadRequest); return
}
authOK := exactAuth(r, chatProfile, readSecret(*chatSecretPath))
optionsOK := *reasoning == "" || (request.ReasoningEffort == *reasoning && request.MaxCompletionTokens == *maxTokens)
state.mu.Lock(); state.Calls++; state.ChatCalls++
if authOK { state.ChatAuthOK++ } else { state.Unauthorized++ }
if optionsOK { state.OptionsOK++ }
state.mu.Unlock()
if !authOK { http.Error(w, "unauthorized", http.StatusUnauthorized); return }
if !optionsOK { http.Error(w, "unexpected options", http.StatusBadRequest); return }
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"id":"chat-smoke","object":"chat.completion","model":%q,"choices":[{"index":0,"message":{"role":"assistant","content":"QUALIFIED"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, *model)
}
registered := map[string]bool{}
register := func(path string, handler http.HandlerFunc) {
if path != "" && !registered[path] { mux.HandleFunc(path, handler); registered[path] = true }
}
register(chatProfile.OperationPath, chatHandler)
if parsed, err := url.Parse(chatProfile.BaseURL); err == nil && parsed.Path != "" {
register(strings.TrimRight(parsed.Path, "/")+chatProfile.OperationPath, chatHandler)
}
// The live self-test overrides the endpoint to a /v1 base while retaining
// the fixture operation /chat/completions.
register("/v1/chat/completions", chatHandler)
var messagesHandler http.HandlerFunc
if *messagesProfileID != "" {
messagesHandler = func(w http.ResponseWriter, r *http.Request) {
var request struct { Model string `json:"model"`; Stream bool `json:"stream"` }
if r.Method != http.MethodPost || json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&request) != nil || request.Model != *model || request.Stream {
http.Error(w, "invalid request", http.StatusBadRequest); return
}
authOK := exactAuth(r, messagesProfile, readSecret(*messagesSecretPath))
state.mu.Lock(); state.Calls++; state.MessagesCalls++
if authOK { state.MessagesAuthOK++ } else { state.Unauthorized++ }
state.mu.Unlock()
if !authOK { http.Error(w, "unauthorized", http.StatusUnauthorized); return }
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"id":"msg-smoke","type":"message","role":"assistant","model":%q,"content":[{"type":"text","text":"QUALIFIED"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`, *model)
}
register(messagesProfile.OperationPath, messagesHandler)
}
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, chatProfile.OperationPath) || strings.HasSuffix(r.URL.Path, "/chat/completions") {
chatHandler(w, r); return
}
if messagesHandler != nil && (strings.HasSuffix(r.URL.Path, messagesProfile.OperationPath) || strings.HasSuffix(r.URL.Path, "/messages")) {
messagesHandler(w, r); return
}
http.NotFound(w, r)
})
server := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5*time.Second}
must(server.ListenAndServe())
}
func inspectDB(path string) {
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)"); must(err)
defer db.Close()
var slots, revisionRows, ciphertextRows int
must(db.QueryRow(`SELECT COUNT(*), SUM(CASE WHEN length(ciphertext) > 16 AND length(nonce) > 0 AND key_id != '' THEN 1 ELSE 0 END) FROM credential_slots`).Scan(&slots, &ciphertextRows))
must(db.QueryRow(`SELECT COUNT(*) FROM credential_slot_revisions`).Scan(&revisionRows))
_ = json.NewEncoder(os.Stdout).Encode(map[string]int{"slots": slots, "revision_rows": revisionRows, "ciphertext_rows": ciphertextRows})
}
func main() {
if len(os.Args) < 2 { must(fmt.Errorf("helper command is required")) }
switch os.Args[1] {
case "material":
if len(os.Args) != 3 { must(fmt.Errorf("material requires a directory")) }
material(os.Args[2])
case "fake": fake(os.Args[2:])
case "inspect-db":
if len(os.Args) != 3 { must(fmt.Errorf("inspect-db requires a path")) }
inspectDB(os.Args[2])
default: must(fmt.Errorf("unknown helper command"))
}
}
GO
}
build_helper() {
write_helper_source
HELPER_BIN="$TMP_DIR/credential-smoke-helper"
(cd "$REPO_ROOT" && go build -o "$HELPER_BIN" "$TMP_DIR/credential_smoke_helper.go")
chmod 700 "$HELPER_BIN"
}
build_runtime_binaries() {
log "building fresh Control Plane, Edge, and Node binaries"
(cd "$REPO_ROOT" && go build -o "$TMP_DIR/control-plane" ./apps/control-plane/cmd/control-plane)
(cd "$REPO_ROOT" && go build -o "$TMP_DIR/iop-edge" ./apps/edge/cmd/edge)
(cd "$REPO_ROOT" && go build -o "$TMP_DIR/iop-node" ./apps/node/cmd/node)
chmod 700 "$TMP_DIR/control-plane" "$TMP_DIR/iop-edge" "$TMP_DIR/iop-node"
}
declare -a USED_PORTS=()
pick_port() {
local output_var="$1" base="$2" attempt=0 candidate used port
while ((attempt < 200)); do
((attempt += 1))
candidate=$((base + RANDOM % 1500))
used=0
for port in "${USED_PORTS[@]}"; do
[ "$port" = "$candidate" ] && used=1
done
[ "$used" = "0" ] || continue
if ! (echo >/dev/tcp/127.0.0.1/"$candidate") 2>/dev/null; then
USED_PORTS+=("$candidate")
printf -v "$output_var" '%s' "$candidate"
return 0
fi
done
die "could not allocate a loopback port"
}
curl_timeout() {
local max_time="$1"
shift
curl --noproxy '*' --connect-timeout "$HTTP_CONNECT_TIMEOUT" --max-time "$max_time" "$@"
}
wait_http() {
local url="$1" label="$2"
shift 2
local deadline=$((SECONDS + 35))
while ! curl_timeout "$HTTP_PROBE_TIMEOUT" -fsS "$@" "$url" >/dev/null 2>&1; do
((SECONDS < deadline)) || die "$label did not become ready"
sleep 0.2
done
}
wait_fake() {
wait_http "http://127.0.0.1:$FAKE_PORT/healthz" "fake provider"
}
start_process() {
local output_var="$1" log_file="$2"
shift 2
"$@" >"$log_file" 2>&1 &
printf -v "$output_var" '%s' "$!"
}
prepare_material() {
"$HELPER_BIN" material "$TMP_DIR"
CHAT_SECRET="$(tr -d '\r\n' <"$TMP_DIR/chat.secret")"
MESSAGES_SECRET="$(tr -d '\r\n' <"$TMP_DIR/messages.secret")"
ROTATED_SECRET="$(tr -d '\r\n' <"$TMP_DIR/rotated.secret")"
cp "$TMP_DIR/chat.secret" "$TMP_DIR/chat-current.secret"
chmod 600 "$TMP_DIR/chat-current.secret"
}
start_fake_provider() {
local chat_profile="$1" messages_profile="$2" model="$3" reasoning="${4:-}" max_tokens="${5:-0}"
local args=(fake --listen "127.0.0.1:$FAKE_PORT" --fixture "$FIXTURE" --chat-profile "$chat_profile" --chat-secret "$TMP_DIR/chat-current.secret" --model "$model")
if [ -n "$messages_profile" ]; then
args+=(--messages-profile "$messages_profile" --messages-secret "$TMP_DIR/messages.secret")
fi
if [ -n "$reasoning" ]; then
args+=(--reasoning-effort "$reasoning" --max-completion-tokens "$max_tokens")
fi
start_process FAKE_PID "$TMP_DIR/fake.log" "$HELPER_BIN" "${args[@]}"
wait_fake
}
write_runtime_configs() {
local chat_profile="$1" messages_profile="$2" upstream_model="$3" chat_provider="$4" messages_provider="$5" provider_endpoint="$6"
local cp_config="$TMP_DIR/control-plane.yaml" edge_config="$TMP_DIR/edge.yaml" node_config="$TMP_DIR/node.yaml"
DB_PATH="$TMP_DIR/credentials.db"
cat >"$cp_config" <<EOF
server:
listen: "127.0.0.1:$CP_HTTP_PORT"
wire_listen: "127.0.0.1:$CP_CLIENT_PORT"
edge_wire_listen: "127.0.0.1:$CP_EDGE_PORT"
edge_wire_tls:
enabled: true
cert: "$TMP_DIR/control-plane.pem"
key: "$TMP_DIR/control-plane.key"
ca: "$TMP_DIR/ca.pem"
peer_role: "edge"
peer_name: "edge-smoke"
database:
url: "$DB_PATH"
credential_encryption:
key_file: "$TMP_DIR/keyring.yaml"
active_key_id: "smoke-key"
active_key_version: 1
credential_plane:
enabled: true
https:
enabled: true
cert: "$TMP_DIR/control-plane.pem"
key: "$TMP_DIR/control-plane.key"
issuer_key_id: "issuer-smoke"
issuer_private_key: "$TMP_DIR/issuer.private"
lease_ttl_seconds: 30
lease_cache_size: 32
logging:
level: "error"
pretty: false
metrics:
port: 0
EOF
cat >"$edge_config" <<EOF
edge:
id: "edge-smoke"
name: "Credential Slot Smoke Edge"
server:
listen: "127.0.0.1:$EDGE_NODE_PORT"
bootstrap:
listen: "127.0.0.1:$EDGE_BOOTSTRAP_PORT"
artifact_dir: "$TMP_DIR/artifacts"
tls:
enabled: true
cert: "$TMP_DIR/edge.pem"
key: "$TMP_DIR/edge.key"
ca: "$TMP_DIR/ca.pem"
peer_role: "node"
peer_name: "node-smoke"
credential_plane:
enabled: true
lease_ttl_seconds: 30
lease_cache_size: 32
control_plane:
enabled: true
wire_addr: "127.0.0.1:$CP_EDGE_PORT"
reconnect_interval_sec: 1
tls:
enabled: true
cert: "$TMP_DIR/edge.pem"
key: "$TMP_DIR/edge.key"
ca: "$TMP_DIR/ca.pem"
server_name: "cp.internal"
peer_role: "control-plane"
peer_name: "cp-smoke"
refresh:
enabled: false
listen: "127.0.0.1:0"
long_context_threshold_tokens: 100000
provider_pool:
max_queue: 2
queue_timeout_ms: 3000
openai:
enabled: true
listen: "127.0.0.1:$EDGE_OPENAI_PORT"
provider_id: "$chat_provider"
session_id: "credential-smoke"
timeout_sec: 40
strict_output: false
tls:
enabled: true
cert: "$TMP_DIR/edge.pem"
key: "$TMP_DIR/edge.key"
a2a:
enabled: false
listen: "127.0.0.1:0"
logging:
level: "error"
pretty: false
path: "$TMP_DIR/edge-runtime.log"
metrics:
port: $EDGE_METRICS_PORT
models:
- id: "credential-smoke-catalog"
context_window_tokens: 4096
providers:
$chat_provider: "$upstream_model"
$messages_provider: "$upstream_model"
nodes:
- id: "node-smoke"
alias: "Credential Smoke Node"
token: "node-registration-token"
providers:
- id: "$chat_provider"
type: "openai_api"
category: "api"
profile: "$chat_profile"
endpoint: "$provider_endpoint"
models: ["$upstream_model"]
health: "available"
capacity: 1
request_timeout_ms: 40000
- id: "$messages_provider"
type: "openai_api"
category: "api"
profile: "$messages_profile"
endpoint: "$provider_endpoint"
models: ["$upstream_model"]
health: "available"
capacity: 1
request_timeout_ms: 40000
runtime:
concurrency: 2
EOF
cat >"$node_config" <<EOF
transport:
edge_addr: "127.0.0.1:$EDGE_NODE_PORT"
token: "node-registration-token"
tls:
enabled: true
cert: "$TMP_DIR/node.pem"
key: "$TMP_DIR/node.key"
ca: "$TMP_DIR/ca.pem"
server_name: "edge.internal"
peer_role: "edge"
peer_name: "edge-smoke"
credential_plane:
enabled: true
recipient_key_id: "recipient-smoke"
recipient_private_key: "$TMP_DIR/recipient.private"
issuer_key_id: "issuer-smoke"
issuer_public_key: "$TMP_DIR/issuer.public"
replay_cache_size: 32
reconnect:
interval_sec: 1
max_attempts: 30
logging:
level: "info"
pretty: false
metrics:
port: 0
node:
id: "node-smoke"
alias: "Credential Smoke Node"
EOF
}
CP_CURL=()
EDGE_CURL=()
CP_BASE_URL=""
EDGE_BASE_URL=""
start_managed_stack() {
log "bootstrapping one principal without exposing its one-time token"
IOP_TOKEN="$("$TMP_DIR/control-plane" --config "$TMP_DIR/control-plane.yaml" principal bootstrap --alias "credential-smoke-principal" 2>>"$TMP_DIR/control-plane.log")"
[ "${#IOP_TOKEN}" = "64" ] || die "principal bootstrap returned an invalid token"
CP_BASE_URL="https://cp-api.internal:$CP_HTTP_PORT"
EDGE_BASE_URL="https://edge-api.internal:$EDGE_OPENAI_PORT"
CP_CURL=(--silent --show-error --cacert "$TMP_DIR/ca.pem" --resolve "cp-api.internal:$CP_HTTP_PORT:127.0.0.1")
EDGE_CURL=(--silent --show-error --cacert "$TMP_DIR/ca.pem" --resolve "edge-api.internal:$EDGE_OPENAI_PORT:127.0.0.1")
start_process CP_PID "$TMP_DIR/control-plane.log" "$TMP_DIR/control-plane" serve --config "$TMP_DIR/control-plane.yaml"
wait_http "$CP_BASE_URL/healthz" "Control Plane HTTPS" "${CP_CURL[@]}"
start_process EDGE_PID "$TMP_DIR/edge.log" "$TMP_DIR/iop-edge" serve --config "$TMP_DIR/edge.yaml"
wait_http "$EDGE_BASE_URL/healthz" "Edge HTTPS ingress" "${EDGE_CURL[@]}"
wait_http "$CP_BASE_URL/edges/edge-smoke" "Edge enrollment" "${CP_CURL[@]}"
start_process NODE_PID "$TMP_DIR/node.log" "$TMP_DIR/iop-node" serve --config "$TMP_DIR/node.yaml"
local deadline=$((SECONDS + 35))
while true; do
if curl_timeout "$HTTP_PROBE_TIMEOUT" -fsS "${CP_CURL[@]}" "$CP_BASE_URL/edges/edge-smoke/status" >"$TMP_DIR/status.json" 2>/dev/null \
&& jq -e '.nodes | any(.[]; .node_id == "node-smoke" and .connected == true)' "$TMP_DIR/status.json" >/dev/null; then
rm -f "$TMP_DIR/status.json"
break
fi
((SECONDS < deadline)) || die "Node did not become dispatch-ready"
sleep 0.2
done
}
json_text() {
local expression="$1"
jq -er "$expression"
}
create_slot_route() {
local prefix="$1" profile_id="$2" vendor="$3" kind="$4" slot_alias="$5" route_alias="$6" provider_id="$7" upstream_model="$8" secret_file="$9"
local slot_response slot_id route_body route_response route_id slot_revision route_revision
slot_response="$(curl_timeout 15 "${CP_CURL[@]}" -fsS -X POST "$CP_BASE_URL/v1/credentials/slots" \
-H "Authorization: Bearer $IOP_TOKEN" \
-H "IOP-Credential-Vendor: $vendor" \
-H "IOP-Credential-Kind: $kind" \
-H "IOP-Credential-Alias: $slot_alias" \
-H 'Content-Type: application/octet-stream' --data-binary "@$secret_file")"
slot_id="$(json_text '(.ID // .id)' <<<"$slot_response")"
route_body="$(jq -cn --arg slot "$slot_id" --arg alias "$route_alias" --arg profile "$profile_id" --arg model "$upstream_model" --arg selector "$provider_id" '{slot_id:$slot,alias:$alias,profile_id:$profile,upstream_model:$model,resource_selector:$selector}')"
route_response="$(curl_timeout 15 "${CP_CURL[@]}" -fsS -X POST "$CP_BASE_URL/v1/credentials/routes" \
-H "Authorization: Bearer $IOP_TOKEN" -H 'Content-Type: application/json' --data-binary "$route_body")"
route_id="$(json_text '(.ID // .id)' <<<"$route_response")"
route_revision="$(json_text '(.Revision // .revision)' <<<"$route_response")"
slot_revision="$(curl_timeout 10 "${CP_CURL[@]}" -fsS "$CP_BASE_URL/v1/credentials/slots" -H "Authorization: Bearer $IOP_TOKEN" \
| jq -er --arg id "$slot_id" '.[] | select((.ID // .id) == $id) | (.Revision // .revision)')"
printf -v "${prefix}_SLOT_ID" '%s' "$slot_id"
printf -v "${prefix}_SLOT_REVISION" '%s' "$slot_revision"
printf -v "${prefix}_ROUTE_ID" '%s' "$route_id"
printf -v "${prefix}_ROUTE_REVISION" '%s' "$route_revision"
}
wait_route_projection() {
local route_id="$1"
local deadline=$((SECONDS + 20))
while true; do
if curl_timeout "$HTTP_PROBE_TIMEOUT" -fsS "${EDGE_CURL[@]}" "$EDGE_BASE_URL/v1/models" -H "Authorization: Bearer $IOP_TOKEN" >"$TMP_DIR/models.json" 2>/dev/null \
&& jq -e --arg id "$route_id" '.data | any(.[]; .id == $id)' "$TMP_DIR/models.json" >/dev/null; then
rm -f "$TMP_DIR/models.json"
return 0
fi
((SECONDS < deadline)) || die "managed route projection did not become visible"
sleep 0.2
done
}
fake_stats() {
curl_timeout "$HTTP_PROBE_TIMEOUT" -fsS "http://127.0.0.1:$FAKE_PORT/stats"
}
request_chat() {
local route_alias="$1" output_file="$2" reasoning="${3:-}" max_tokens="${4:-0}"
local body code
body="$(jq -cn --arg model "$route_alias" --arg reasoning "$reasoning" --argjson cap "$max_tokens" '
{model:$model,messages:[{role:"user",content:"credential qualification"}],stream:false}
+ (if $reasoning == "" then {} else {reasoning_effort:$reasoning,max_completion_tokens:$cap} end)
')"
code="$(curl_timeout "$HTTP_COMPLETION_TIMEOUT" "${EDGE_CURL[@]}" -o "$output_file" -w '%{http_code}' \
-X POST "$EDGE_BASE_URL/v1/chat/completions" -H "Authorization: Bearer $IOP_TOKEN" -H 'Content-Type: application/json' --data-binary "$body")"
[ "$code" = "200" ] || die "Chat request returned HTTP $code"
jq -e '.choices[0].finish_reason == "stop" and (.choices[0].message.content | type == "string" and length > 0)' "$output_file" >/dev/null \
|| die "Chat response failed structural validation"
rm -f "$output_file"
}
request_messages() {
local route_alias="$1" output_file="$2" body code
body="$(jq -cn --arg model "$route_alias" '{model:$model,max_tokens:64,messages:[{role:"user",content:"credential qualification"}],stream:false}')"
code="$(curl_timeout "$HTTP_COMPLETION_TIMEOUT" "${EDGE_CURL[@]}" -o "$output_file" -w '%{http_code}' \
-X POST "$EDGE_BASE_URL/v1/messages" -H "x-api-key: $IOP_TOKEN" -H 'anthropic-version: 2023-06-01' -H 'Content-Type: application/json' --data-binary "$body")"
[ "$code" = "200" ] || die "Messages request returned HTTP $code"
jq -e '.type == "message" and .stop_reason == "end_turn" and ([.content[] | select(.type == "text") | .text] | join("") | length > 0)' "$output_file" >/dev/null \
|| die "Messages response failed structural validation"
rm -f "$output_file"
}
expect_tls_client_rejected() {
local label="$1" port="$2" cert="$3" key="$4" server_name="$5" output
output="$TMP_DIR/tls-$label.log"
local -a args=(-brief -connect "127.0.0.1:$port" -servername "$server_name" -CAfile "$TMP_DIR/ca.pem")
if [ -n "$cert" ]; then
args+=(-cert "$cert" -key "$key")
fi
if timeout 5 openssl s_client "${args[@]}" </dev/null >"$output" 2>&1; then
rg -qi 'alert|certificate required|handshake failure|peer workload identity mismatch' "$output" \
|| die "$label unexpectedly completed an authenticated TLS handshake"
fi
}
assert_tls_boundaries() {
local code
if curl_timeout 2 -fsS "http://127.0.0.1:$CP_HTTP_PORT/healthz" >/dev/null 2>&1; then
die "credential HTTPS listener accepted plaintext HTTP"
fi
if curl_timeout 3 -fsS --cacert "$TMP_DIR/other-ca.pem" --resolve "cp-api.internal:$CP_HTTP_PORT:127.0.0.1" "$CP_BASE_URL/healthz" >/dev/null 2>&1; then
die "credential HTTPS listener accepted an unrelated CA"
fi
code="$(curl_timeout 3 "${CP_CURL[@]}" -o /dev/null -w '%{http_code}' "$CP_BASE_URL/v1/credentials/slots")"
[ "$code" = "401" ] || die "credential HTTPS operation without principal bearer returned HTTP $code"
expect_tls_client_rejected "cp-no-cert" "$CP_EDGE_PORT" "" "" "cp.internal"
expect_tls_client_rejected "cp-wrong-peer" "$CP_EDGE_PORT" "$TMP_DIR/wrong.pem" "$TMP_DIR/wrong.key" "cp.internal"
expect_tls_client_rejected "edge-no-cert" "$EDGE_NODE_PORT" "" "" "edge.internal"
expect_tls_client_rejected "edge-wrong-peer" "$EDGE_NODE_PORT" "$TMP_DIR/wrong.pem" "$TMP_DIR/wrong.key" "edge.internal"
rm -f "$TMP_DIR"/tls-*.log
}
scan_sensitive_artifacts() {
local -a candidates=(
"$TMP_DIR/control-plane.yaml" "$TMP_DIR/edge.yaml" "$TMP_DIR/node.yaml"
"$TMP_DIR/control-plane.log" "$TMP_DIR/edge.log" "$TMP_DIR/edge-runtime.log" "$TMP_DIR/node.log" "$TMP_DIR/fake.log"
"$TMP_DIR/credentials.db"
)
local -a files=()
local value file
for file in "${candidates[@]}"; do
[ -f "$file" ] && files+=("$file")
done
for value in "$IOP_TOKEN" "$CHAT_SECRET" "$MESSAGES_SECRET" "$ROTATED_SECRET"; do
if rg -a -F -l -- "$value" "${files[@]}" >/dev/null 2>&1; then
die "sensitive value appeared in a persisted or captured artifact"
fi
done
local private_key_pattern='BEGIN (AGE |OPENSSH |RSA |EC |ENCRYPTED )?PRIVATE KEY|AGE-SECRET-KEY-'
printf '%s\n' '-----BEGIN PRIVATE KEY-----' | rg -a -q "$private_key_pattern" \
|| die "private-key evidence matcher rejected its positive control"
if rg -a -n "$private_key_pattern|lease[_-]?id|credential qualification|QUALIFIED" \
"${files[@]}" >/dev/null 2>&1; then
die "private key, lease id, prompt, or provider body appeared in config/log evidence"
fi
}
run_deterministic() {
require_tools
setup_workspace
validate_fixture
build_helper
prepare_material
build_runtime_binaries
local chat_profile messages_profile chat_vendor messages_vendor chat_kind messages_kind
local upstream_model chat_provider messages_provider chat_slot_alias messages_slot_alias chat_route_alias messages_route_alias
chat_profile="$(jq -r '.routes[] | select(.id == "chat-slot-route") | .profile_id' "$FIXTURE")"
messages_profile="$(jq -r '.routes[] | select(.id == "messages-slot-route") | .profile_id' "$FIXTURE")"
chat_vendor="$(jq -r --arg id "$chat_profile" '.profiles[] | select(.id == $id) | .vendor' "$FIXTURE")"
messages_vendor="$(jq -r --arg id "$messages_profile" '.profiles[] | select(.id == $id) | .vendor' "$FIXTURE")"
chat_kind="$(jq -r --arg id "$chat_profile" '.profiles[] | select(.id == $id) | .credential_kind' "$FIXTURE")"
messages_kind="$(jq -r --arg id "$messages_profile" '.profiles[] | select(.id == $id) | .credential_kind' "$FIXTURE")"
upstream_model="$(jq -r '.routes[0].upstream_model' "$FIXTURE")"
chat_provider="$(jq -r '.routes[] | select(.id == "chat-slot-route") | .provider_id' "$FIXTURE")"
messages_provider="$(jq -r '.routes[] | select(.id == "messages-slot-route") | .provider_id' "$FIXTURE")"
chat_slot_alias="$(jq -r '.routes[] | select(.id == "chat-slot-route") | .slot_alias' "$FIXTURE")"
messages_slot_alias="$(jq -r '.routes[] | select(.id == "messages-slot-route") | .slot_alias' "$FIXTURE")"
chat_route_alias="$(jq -r '.routes[] | select(.id == "chat-slot-route") | .route_alias' "$FIXTURE")"
messages_route_alias="$(jq -r '.routes[] | select(.id == "messages-slot-route") | .route_alias' "$FIXTURE")"
pick_port FAKE_PORT 41000
pick_port CP_HTTP_PORT 25000
pick_port CP_CLIENT_PORT 26500
pick_port CP_EDGE_PORT 28000
pick_port EDGE_NODE_PORT 30000
pick_port EDGE_BOOTSTRAP_PORT 32000
pick_port EDGE_OPENAI_PORT 34000
pick_port EDGE_METRICS_PORT 36000
start_fake_provider "$chat_profile" "$messages_profile" "$upstream_model"
write_runtime_configs "$chat_profile" "$messages_profile" "$upstream_model" "$chat_provider" "$messages_provider" "http://127.0.0.1:$FAKE_PORT"
start_managed_stack
assert_tls_boundaries
create_slot_route CHAT "$chat_profile" "$chat_vendor" "$chat_kind" "$chat_slot_alias" "$chat_route_alias" "$chat_provider" "$upstream_model" "$TMP_DIR/chat.secret"
create_slot_route MESSAGES "$messages_profile" "$messages_vendor" "$messages_kind" "$messages_slot_alias" "$messages_route_alias" "$messages_provider" "$upstream_model" "$TMP_DIR/messages.secret"
wait_route_projection "$CHAT_ROUTE_ID"
wait_route_projection "$MESSAGES_ROUTE_ID"
request_chat "$chat_route_alias" "$TMP_DIR/chat.response.json"
request_messages "$messages_route_alias" "$TMP_DIR/messages.response.json"
local stats initial_chat_auth initial_messages_auth
stats="$(fake_stats)"
initial_chat_auth="$(jq -r '.chat_auth_ok' <<<"$stats")"
initial_messages_auth="$(jq -r '.messages_auth_ok' <<<"$stats")"
[ "$initial_chat_auth" = "1" ] && [ "$initial_messages_auth" = "1" ] && [ "$(jq -r '.unauthorized' <<<"$stats")" = "0" ] \
|| die "fake provider did not observe exact Chat and Messages authorization"
local rotate_response rotated_revision
rotate_response="$(curl_timeout 15 "${CP_CURL[@]}" -fsS -X POST "$CP_BASE_URL/v1/credentials/slots/$CHAT_SLOT_ID/rotate" \
-H "Authorization: Bearer $IOP_TOKEN" -H "IOP-Expected-Revision: $CHAT_SLOT_REVISION" \
-H 'Content-Type: application/octet-stream' --data-binary "@$TMP_DIR/rotated.secret")"
rotated_revision="$(json_text '(.Revision // .revision)' <<<"$rotate_response")"
cp "$TMP_DIR/rotated.secret" "$TMP_DIR/chat-current.secret.next"
chmod 600 "$TMP_DIR/chat-current.secret.next"
mv "$TMP_DIR/chat-current.secret.next" "$TMP_DIR/chat-current.secret"
request_chat "$chat_route_alias" "$TMP_DIR/chat-rotated.response.json"
stats="$(fake_stats)"
[ "$(jq -r '.chat_calls' <<<"$stats")" = "2" ] && [ "$(jq -r '.chat_auth_ok' <<<"$stats")" = "2" ] \
|| die "rotated credential revision did not reach the exact Chat auth boundary"
wait_http "http://127.0.0.1:$EDGE_METRICS_PORT/healthz" "Edge metrics"
curl_timeout 5 -fsS "http://127.0.0.1:$EDGE_METRICS_PORT/metrics" >"$TMP_DIR/metrics.txt"
rg -F "credential_slot_ref=\"$CHAT_SLOT_ID\"" "$TMP_DIR/metrics.txt" >/dev/null || die "Chat slot attribution is missing"
rg -F "credential_revision=\"$CHAT_SLOT_REVISION\"" "$TMP_DIR/metrics.txt" >/dev/null || die "initial credential revision attribution is missing"
rg -F "credential_revision=\"$rotated_revision\"" "$TMP_DIR/metrics.txt" >/dev/null || die "rotated credential revision attribution is missing"
for value in "$IOP_TOKEN" "$CHAT_SECRET" "$MESSAGES_SECRET" "$ROTATED_SECRET"; do
! rg -F -- "$value" "$TMP_DIR/metrics.txt" >/dev/null || die "sensitive value appeared in metrics"
done
! rg -n 'lease[_-]?id|BEGIN .*PRIVATE KEY|credential qualification|QUALIFIED' "$TMP_DIR/metrics.txt" >/dev/null \
|| die "unsafe attribution dimension appeared in metrics"
rm -f "$TMP_DIR/metrics.txt"
local db_stats
db_stats="$("$HELPER_BIN" inspect-db "$DB_PATH")"
[ "$(jq -r '.slots' <<<"$db_stats")" = "2" ] \
&& [ "$(jq -r '.revision_rows' <<<"$db_stats")" = "3" ] \
&& [ "$(jq -r '.ciphertext_rows' <<<"$db_stats")" = "2" ] \
|| die "SQLite ciphertext/revision inspection failed: $db_stats"
for value in "$CHAT_SECRET" "$MESSAGES_SECRET" "$ROTATED_SECRET"; do
! rg -a -F -- "$value" "$DB_PATH" >/dev/null || die "provider plaintext appeared in SQLite"
done
local pre_revoke_stats pre_revoke_tunnels revoke_response revoked_revision revoked_code post_revoke_stats post_revoke_tunnels
pre_revoke_stats="$(fake_stats)"
pre_revoke_tunnels="$(rg -c 'provider tunnel request received' "$TMP_DIR/node.log" || true)"
revoke_response="$(curl_timeout 15 "${CP_CURL[@]}" -fsS -X POST "$CP_BASE_URL/v1/credentials/slots/$CHAT_SLOT_ID/revoke" \
-H "Authorization: Bearer $IOP_TOKEN" -H "IOP-Expected-Revision: $rotated_revision")"
revoked_revision="$(json_text '(.Revision // .revision)' <<<"$revoke_response")"
revoked_code="$(curl_timeout "$HTTP_COMPLETION_TIMEOUT" "${EDGE_CURL[@]}" -o "$TMP_DIR/revoked.response.json" -w '%{http_code}' \
-X POST "$EDGE_BASE_URL/v1/chat/completions" -H "Authorization: Bearer $IOP_TOKEN" -H 'Content-Type: application/json' \
--data-binary "$(jq -cn --arg model "$chat_route_alias" '{model:$model,messages:[{role:"user",content:"credential qualification"}],stream:false}')")"
[[ "$revoked_code" != 2* ]] || die "post-revoke request unexpectedly succeeded"
rm -f "$TMP_DIR/revoked.response.json"
sleep 0.3
post_revoke_stats="$(fake_stats)"
post_revoke_tunnels="$(rg -c 'provider tunnel request received' "$TMP_DIR/node.log" || true)"
[ "$pre_revoke_stats" = "$post_revoke_stats" ] || die "post-revoke request changed upstream counters"
[ "$pre_revoke_tunnels" = "$post_revoke_tunnels" ] || die "post-revoke request reached Node lease consumption"
[ "$(jq -r '.messages_calls' <<<"$post_revoke_stats")" = "1" ] || die "post-revoke request fell back to the same-model Messages slot"
scan_sensitive_artifacts
jq -cn \
--arg chat_profile "$chat_profile" --arg messages_profile "$messages_profile" \
--arg chat_slot "$CHAT_SLOT_ID" --arg messages_slot "$MESSAGES_SLOT_ID" \
--argjson initial_revision "$CHAT_SLOT_REVISION" --argjson rotated_revision "$rotated_revision" --argjson revoked_revision "$revoked_revision" \
--argjson upstream_calls "$(jq -r '.calls' <<<"$post_revoke_stats")" --argjson lease_delivery_attempts "$post_revoke_tunnels" \
'{mode:"deterministic",profiles:[$chat_profile,$messages_profile],same_model_two_slot:true,exact_auth:{chat:2,messages:1},ciphertext_only:true,attribution:{chat_slot_ref:$chat_slot,messages_slot_ref:$messages_slot,initial_revision:$initial_revision,rotated_revision:$rotated_revision},tls_negative_matrix:"passed",post_revoke:{revision:$revoked_revision,lease_delivery_attempts:$lease_delivery_attempts,upstream_calls:$upstream_calls,counters_unchanged:true,no_fallback:true},result:"success"}'
}
run_live() {
QUIET=1
require_tools
setup_workspace
validate_fixture
build_helper
prepare_material
build_runtime_binaries
local profile_json vendor kind driver base_url operation_path endpoint provider_id route_alias slot_alias secret_line
profile_json="$(jq -ec --arg id "$LIVE_PROFILE" '.profiles[] | select(.id == $id)' "$FIXTURE")" || die "live profile is not present in the fixture"
vendor="$(jq -r '.vendor' <<<"$profile_json")"
kind="$(jq -r '.credential_kind' <<<"$profile_json")"
driver="$(jq -r '.driver' <<<"$profile_json")"
base_url="$(jq -r '.base_url' <<<"$profile_json")"
operation_path="$(jq -r '.operation_path' <<<"$profile_json")"
[ "$driver" = "openai_chat" ] || die "live one-shot currently requires an openai_chat profile"
[ "$operation_path" = "/chat/completions" ] || die "live profile operation path is not approved"
endpoint="$base_url"
if [ "${IOP_CREDENTIAL_SLOT_LIVE_TEST:-0}" = "1" ]; then
[ -n "${IOP_CREDENTIAL_SLOT_LIVE_TEST_ENDPOINT:-}" ] || die "live self-test endpoint is missing"
endpoint="$IOP_CREDENTIAL_SLOT_LIVE_TEST_ENDPOINT"
fi
[[ "$endpoint" == https://* ]] || [ "${IOP_CREDENTIAL_SLOT_LIVE_TEST:-0}" = "1" ] \
|| die "live provider endpoint must use HTTPS"
secret_line=""
IFS= read -r secret_line || [ -n "$secret_line" ] || die "live secret stdin was empty"
[ -n "$secret_line" ] || die "live secret stdin was empty"
printf '%s' "$secret_line" >"$TMP_DIR/live.secret"
chmod 600 "$TMP_DIR/live.secret"
CHAT_SECRET="$secret_line"
cp "$TMP_DIR/live.secret" "$TMP_DIR/chat-current.secret"
secret_line=""
pick_port CP_HTTP_PORT 25000
pick_port CP_CLIENT_PORT 26500
pick_port CP_EDGE_PORT 28000
pick_port EDGE_NODE_PORT 30000
pick_port EDGE_BOOTSTRAP_PORT 32000
pick_port EDGE_OPENAI_PORT 34000
pick_port EDGE_METRICS_PORT 36000
provider_id="credential-live-provider"
route_alias="credential-live-route"
slot_alias="credential-live-slot"
# A second unreachable provider is present only to keep the shared config
# shape stable; the explicit resource selector makes it unreachable.
write_runtime_configs "$LIVE_PROFILE" "$LIVE_PROFILE" "$LIVE_MODEL" "$provider_id" "credential-live-unselected" "$endpoint"
start_managed_stack
create_slot_route LIVE "$LIVE_PROFILE" "$vendor" "$kind" "$slot_alias" "$route_alias" "$provider_id" "$LIVE_MODEL" "$TMP_DIR/live.secret"
wait_route_projection "$LIVE_ROUTE_ID"
local body live_code
body="$(jq -cn --arg model "$route_alias" --arg effort "$LIVE_REASONING_EFFORT" --argjson cap "$LIVE_MAX_COMPLETION_TOKENS" \
'{model:$model,messages:[{role:"user",content:"Reply with one word: qualified"}],stream:false,reasoning_effort:$effort,max_completion_tokens:$cap}')"
live_code="$(curl_timeout "$HTTP_COMPLETION_TIMEOUT" "${EDGE_CURL[@]}" -o "$TMP_DIR/live.response.json" -w '%{http_code}' \
-X POST "$EDGE_BASE_URL/v1/chat/completions" -H "Authorization: Bearer $IOP_TOKEN" -H 'Content-Type: application/json' --data-binary "$body" \
2>"$TMP_DIR/live.curl.stderr")"
[ "$live_code" = "200" ] || die "live provider request returned HTTP $live_code"
jq -e '.choices[0].finish_reason == "stop" and (.choices[0].message.content | type == "string" and length > 0)' "$TMP_DIR/live.response.json" >/dev/null \
|| die "live provider response failed structural validation"
rm -f "$TMP_DIR/live.response.json"
[ ! -s "$TMP_DIR/live.curl.stderr" ] || die "live provider request produced stderr"
rm -f "$TMP_DIR/live.curl.stderr"
local live_revision
live_revision="$(curl_timeout 10 "${CP_CURL[@]}" -fsS "$CP_BASE_URL/v1/credentials/slots" -H "Authorization: Bearer $IOP_TOKEN" \
| jq -er --arg id "$LIVE_SLOT_ID" '.[] | select((.ID // .id) == $id) | (.Revision // .revision)')"
scan_sensitive_artifacts
jq -cn --arg provider "$LIVE_PROFILE" --arg model "$LIVE_MODEL" --arg slot "$LIVE_SLOT_ID" \
--argjson revision "$live_revision" --arg date "$(date -u +%Y-%m-%d)" \
'{provider:$provider,model:$model,credential_slot_ref:$slot,credential_revision:$revision,date:$date,result:"success"}'
}
run_live_gate_self_test() {
require_tools
setup_workspace
validate_fixture
build_helper
prepare_material
pick_port FAKE_PORT 41000
start_fake_provider "openai" "" "gpt-5.6-luna" "high" 128
local before after fifo unread child_count_before child_count_after
before="$(fake_stats)"
fifo="$TMP_DIR/unread.fifo"
mkfifo "$fifo"
exec 9<>"$fifo"
printf '%s\n' 'LIVE_GATE_INPUT_MUST_REMAIN_UNREAD' >&9
child_count_before="$(find "$TMP_DIR" -maxdepth 1 -type d -name 'iop-credential-slot-smoke.*' | wc -l | tr -d ' ')"
if TMPDIR="$TMP_DIR" IOP_CREDENTIAL_SLOT_SMOKE_KEEP_TMP=0 IOP_ALLOW_LIVE_PROVIDER=0 IOP_CREDENTIAL_SLOT_LIVE_TEST=1 \
IOP_CREDENTIAL_SLOT_LIVE_TEST_ENDPOINT="http://127.0.0.1:$FAKE_PORT/v1" \
"$0" --live --secret-stdin --profile openai --model gpt-5.6-luna --reasoning-effort high --max-completion-tokens 128 \
<&9 >"$TMP_DIR/no-opt-in.stdout" 2>"$TMP_DIR/no-opt-in.stderr"; then
die "live gate accepted a request without opt-in"
fi
IFS= read -r unread <&9
exec 9>&-
rm -f "$fifo"
[ "$unread" = "LIVE_GATE_INPUT_MUST_REMAIN_UNREAD" ] || die "live gate read stdin before rejecting opt-in"
child_count_after="$(find "$TMP_DIR" -maxdepth 1 -type d -name 'iop-credential-slot-smoke.*' | wc -l | tr -d ' ')"
[ "$child_count_before" = "$child_count_after" ] || die "live gate created a workspace before rejecting opt-in"
rg -F 'live mode requires IOP_ALLOW_LIVE_PROVIDER=1' "$TMP_DIR/no-opt-in.stderr" >/dev/null \
|| die "live gate did not emit the expected early rejection"
[ ! -s "$TMP_DIR/no-opt-in.stdout" ] || die "live gate emitted stdout before rejecting opt-in"
after="$(fake_stats)"
[ "$before" = "$after" ] || die "live gate contacted the provider without opt-in"
rm -f "$TMP_DIR/no-opt-in.stdout" "$TMP_DIR/no-opt-in.stderr"
TMPDIR="$TMP_DIR" IOP_CREDENTIAL_SLOT_SMOKE_KEEP_TMP=0 IOP_ALLOW_LIVE_PROVIDER=1 IOP_CREDENTIAL_SLOT_LIVE_TEST=1 \
IOP_CREDENTIAL_SLOT_LIVE_TEST_ENDPOINT="http://127.0.0.1:$FAKE_PORT/v1" \
"$0" --live --secret-stdin --profile openai --model gpt-5.6-luna --reasoning-effort high --max-completion-tokens 128 \
<"$TMP_DIR/chat.secret" >"$TMP_DIR/authorized.stdout" 2>"$TMP_DIR/authorized.stderr"
[ ! -s "$TMP_DIR/authorized.stderr" ] || die "authorized live self-test emitted stderr"
jq -e '
(keys | sort) == ["credential_revision","credential_slot_ref","date","model","provider","result"] and
.provider == "openai" and .model == "gpt-5.6-luna" and .credential_revision == 1 and .result == "success"
' "$TMP_DIR/authorized.stdout" >/dev/null || die "authorized live self-test emitted an invalid sanitized record"
after="$(fake_stats)"
[ "$(jq -r '.calls' <<<"$after")" = "1" ] \
&& [ "$(jq -r '.chat_auth_ok' <<<"$after")" = "1" ] \
&& [ "$(jq -r '.options_ok' <<<"$after")" = "1" ] \
&& [ "$(jq -r '.unauthorized' <<<"$after")" = "0" ] \
|| die "authorized live self-test did not make exactly one correct upstream call"
! rg -F -- "$CHAT_SECRET" "$TMP_DIR/authorized.stdout" "$TMP_DIR/authorized.stderr" >/dev/null \
|| die "live self-test leaked its provider secret"
rm -f "$TMP_DIR/authorized.stdout" "$TMP_DIR/authorized.stderr"
jq -cn '{mode:"self-test-live-gate",zero_io_without_opt_in:true,stdin_unread_without_opt_in:true,authorized_upstream_calls:1,retries:0,forwarded_options:true,sanitized_output:true,result:"success"}'
}
case "$MODE" in
deterministic) run_deterministic ;;
self-test-live-gate) run_live_gate_self_test ;;
live) run_live ;;
*) early_die "invalid mode" ;;
esac
Reference in a new issue View git blame Copy permalink