313 lines
10 KiB
Go
313 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestSmokeOpenAICommandSuccess(t *testing.T) {
|
|
// Record the endpoint hit order so we assert health -> models -> responses.
|
|
var calledEndpoints []string
|
|
// Start an httptest server to mock the OpenAI-compatible endpoint
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calledEndpoints = append(calledEndpoints, r.URL.Path)
|
|
switch r.URL.Path {
|
|
case "/healthz":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
case "/v1/models":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"object":"list","data":[{"id":"test-model","object":"model","created":123456,"owned_by":"iop"}]}`))
|
|
case "/v1/responses":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"id":"resp-123","object":"response","created_at":123456,"model":"test-model","output_text":"responses pong","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"responses pong"}]}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`))
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
root := rootCmd()
|
|
var out bytes.Buffer
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server.URL, "--prompt", "ping", "--timeout", "5s"})
|
|
if err := root.Execute(); err != nil {
|
|
t.Fatalf("smoke openai command failed: %v\n%s", err, out.String())
|
|
}
|
|
|
|
got := out.String()
|
|
wants := []string{
|
|
"Step 1: Checking /healthz ... [OK]",
|
|
"Step 2: Checking /v1/models ... [OK]",
|
|
"Step 3: Checking /v1/responses ... [OK]",
|
|
"Responses Output Text: \"responses pong\"",
|
|
"IOP Edge OpenAI Smoke Test SUCCESS!",
|
|
}
|
|
for _, w := range wants {
|
|
if !strings.Contains(got, w) {
|
|
t.Errorf("smoke openai success output missing: %q\nFull output:\n%s", w, got)
|
|
}
|
|
}
|
|
|
|
wantOrder := []string{"/healthz", "/v1/models", "/v1/responses"}
|
|
if len(calledEndpoints) != len(wantOrder) {
|
|
t.Fatalf("expected endpoints %v, got %v", wantOrder, calledEndpoints)
|
|
}
|
|
for i, want := range wantOrder {
|
|
if calledEndpoints[i] != want {
|
|
t.Fatalf("expected endpoint order %v, got %v", wantOrder, calledEndpoints)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSmokeOpenAICommandAuthenticatedEndpoint(t *testing.T) {
|
|
oldAPIKey, hadAPIKey := os.LookupEnv("OPENAI_API_KEY")
|
|
os.Unsetenv("OPENAI_API_KEY")
|
|
defer func() {
|
|
if hadAPIKey {
|
|
os.Setenv("OPENAI_API_KEY", oldAPIKey)
|
|
} else {
|
|
os.Unsetenv("OPENAI_API_KEY")
|
|
}
|
|
}()
|
|
|
|
const token = "test-token"
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/healthz":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
case "/v1/models", "/v1/responses":
|
|
if r.Header.Get("Authorization") != "Bearer "+token {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
w.Write([]byte(`{"error":{"type":"unauthorized","message":"missing bearer token"}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
if r.URL.Path == "/v1/models" {
|
|
w.Write([]byte(`{"object":"list","data":[{"id":"test-model","object":"model","created":123456,"owned_by":"iop"}]}`))
|
|
return
|
|
}
|
|
w.Write([]byte(`{"id":"resp-123","object":"response","created_at":123456,"model":"test-model","output_text":"responses pong","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"responses pong"}]}]}`))
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
root := rootCmd()
|
|
var out bytes.Buffer
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server.URL, "--timeout", "5s"})
|
|
err := root.Execute()
|
|
if err == nil {
|
|
t.Fatalf("expected unauthenticated smoke to fail\n%s", out.String())
|
|
}
|
|
if !strings.Contains(err.Error(), "401") {
|
|
t.Fatalf("expected 401 error, got %v\n%s", err, out.String())
|
|
}
|
|
|
|
os.Setenv("OPENAI_API_KEY", token)
|
|
root = rootCmd()
|
|
out.Reset()
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server.URL, "--timeout", "5s"})
|
|
if err := root.Execute(); err != nil {
|
|
t.Fatalf("authenticated smoke failed: %v\n%s", err, out.String())
|
|
}
|
|
if !strings.Contains(out.String(), "IOP Edge OpenAI Smoke Test SUCCESS!") {
|
|
t.Fatalf("success output missing:\n%s", out.String())
|
|
}
|
|
}
|
|
|
|
func TestSmokeOpenAICommandWorkspace(t *testing.T) {
|
|
// Create a temporary directory for workspace
|
|
tmpDir, err := os.MkdirTemp("", "iop-smoke-workspace-*")
|
|
if err != nil {
|
|
t.Fatalf("failed to create temp dir: %v", err)
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
// Create an expect file inside the workspace
|
|
expectFile := "test_marker.txt"
|
|
expectContent := "hello smoke test"
|
|
if err := os.WriteFile(filepath.Join(tmpDir, expectFile), []byte(expectContent), 0644); err != nil {
|
|
t.Fatalf("failed to write expect file: %v", err)
|
|
}
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/healthz":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
case "/v1/models":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"object":"list","data":[{"id":"test-model","object":"model","created":123456,"owned_by":"iop"}]}`))
|
|
case "/v1/responses":
|
|
// Assert request body
|
|
bodyBytes, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
t.Errorf("failed to read request body: %v", err)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
var reqPayload map[string]interface{}
|
|
if err := json.Unmarshal(bodyBytes, &reqPayload); err != nil {
|
|
t.Errorf("failed to unmarshal request body: %v", err)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Validate metadata.workspace
|
|
metadata, ok := reqPayload["metadata"].(map[string]interface{})
|
|
if !ok {
|
|
t.Errorf("metadata missing in request body")
|
|
} else {
|
|
gotWS := metadata["workspace"]
|
|
if gotWS != tmpDir {
|
|
t.Errorf("expected workspace %q, got %q", tmpDir, gotWS)
|
|
}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"id":"resp-123","object":"response","created_at":123456,"model":"test-model","output_text":"responses pong","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"responses pong"}]}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`))
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
root := rootCmd()
|
|
var out bytes.Buffer
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{
|
|
"smoke", "openai",
|
|
"--model", "test-model",
|
|
"--base-url", server.URL,
|
|
"--prompt", "ping",
|
|
"--timeout", "5s",
|
|
"--workspace", tmpDir,
|
|
"--expect-file", expectFile,
|
|
"--expect-contains", "smoke test",
|
|
})
|
|
if err := root.Execute(); err != nil {
|
|
t.Fatalf("smoke openai command failed: %v\n%s", err, out.String())
|
|
}
|
|
|
|
got := out.String()
|
|
wants := []string{
|
|
"Step 1: Checking /healthz ... [OK]",
|
|
"Step 2: Checking /v1/models ... [OK]",
|
|
"Step 3: Checking /v1/responses ... [OK]",
|
|
"Responses Output Text: \"responses pong\"",
|
|
"IOP Edge OpenAI Smoke Test SUCCESS!",
|
|
}
|
|
for _, w := range wants {
|
|
if !strings.Contains(got, w) {
|
|
t.Errorf("smoke openai success output missing: %q\nFull output:\n%s", w, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSmokeOpenAICommandFailure(t *testing.T) {
|
|
// 1. Non-200 /healthz error
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/healthz" {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
w.Write([]byte("internal server error"))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer server.Close()
|
|
|
|
root := rootCmd()
|
|
var out bytes.Buffer
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server.URL})
|
|
err := root.Execute()
|
|
if err == nil {
|
|
t.Fatalf("expected error from non-200 healthz, got nil. Output:\n%s", out.String())
|
|
}
|
|
if !strings.Contains(err.Error(), "returned non-200 status 500") {
|
|
t.Errorf("unexpected error message: %v", err)
|
|
}
|
|
|
|
// 2. Responses non-200 error
|
|
server4 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/healthz":
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
case "/v1/models":
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"object":"list","data":[{"id":"test-model","object":"model"}]}`))
|
|
case "/v1/responses":
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
w.Write([]byte("node offline"))
|
|
}
|
|
}))
|
|
defer server4.Close()
|
|
|
|
root = rootCmd()
|
|
out.Reset()
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server4.URL})
|
|
err4 := root.Execute()
|
|
if err4 == nil {
|
|
t.Fatalf("expected error from non-200 responses, got nil")
|
|
}
|
|
if !strings.Contains(err4.Error(), "returned non-200 status 503") {
|
|
t.Errorf("unexpected error message: %v", err4)
|
|
}
|
|
|
|
// 3. Responses empty text error
|
|
server5 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.URL.Path {
|
|
case "/healthz":
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"status":"ok"}`))
|
|
case "/v1/models":
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"object":"list","data":[{"id":"test-model","object":"model"}]}`))
|
|
case "/v1/responses":
|
|
w.WriteHeader(http.StatusOK)
|
|
w.Write([]byte(`{"id":"resp-123","object":"response","created_at":123456,"model":"test-model","output_text":"","output":[]}`))
|
|
}
|
|
}))
|
|
defer server5.Close()
|
|
|
|
root = rootCmd()
|
|
out.Reset()
|
|
root.SetOut(&out)
|
|
root.SetErr(&out)
|
|
root.SetArgs([]string{"smoke", "openai", "--model", "test-model", "--base-url", server5.URL})
|
|
err5 := root.Execute()
|
|
if err5 == nil {
|
|
t.Fatalf("expected error from empty responses text, got nil")
|
|
}
|
|
if !strings.Contains(err5.Error(), "returned empty response text") {
|
|
t.Errorf("unexpected error message: %v", err5)
|
|
}
|
|
}
|