toki
46bd8563e2
- agent-contract: OpenAI compatible API 계약 문서 업데이트 - apps/edge: OpenAI API smoke/test 스크립트 및 핸들러 개선 - scripts: e2e 테스트 스크립트 업데이트
332 lines
11 KiB
Go
332 lines
11 KiB
Go
package edgecmd
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func smokeCmd() *cobra.Command {
|
|
c := &cobra.Command{
|
|
Use: "smoke",
|
|
Short: "Run verification and smoke tests",
|
|
Long: `Run diagnostic smoke tests to verify the connectivity and functionality of IOP Edge components.
|
|
|
|
Typical local flow:
|
|
1. iop-edge config init
|
|
2. iop-edge node register <node-id>
|
|
3. iop-edge serve
|
|
4. iop-edge smoke openai --model <model>`,
|
|
Example: ` iop-edge smoke openai --model gemma4:26b
|
|
iop-edge smoke openai --model gemma4:26b --base-url http://127.0.0.1:18081`,
|
|
}
|
|
c.AddCommand(smokeOpenAICmd())
|
|
return c
|
|
}
|
|
|
|
var (
|
|
smokeModel string
|
|
smokeBaseURL string
|
|
smokePrompt string
|
|
smokeTimeout string
|
|
smokeWorkspace string
|
|
smokeExpectFile string
|
|
smokeExpectContains string
|
|
smokeAPIKey string
|
|
smokeAPIKeyFile string
|
|
)
|
|
|
|
type openAIModel struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Created int64 `json:"created"`
|
|
OwnedBy string `json:"owned_by"`
|
|
}
|
|
|
|
type openAIModelsResponse struct {
|
|
Object string `json:"object"`
|
|
Data []openAIModel `json:"data"`
|
|
}
|
|
|
|
type responsesContentItem struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
type responsesOutputItem struct {
|
|
Type string `json:"type"`
|
|
Role string `json:"role"`
|
|
Content []responsesContentItem `json:"content"`
|
|
}
|
|
|
|
type responsesResponse struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
Model string `json:"model"`
|
|
OutputText string `json:"output_text"`
|
|
Output []responsesOutputItem `json:"output"`
|
|
}
|
|
|
|
func (r responsesResponse) text() string {
|
|
if strings.TrimSpace(r.OutputText) != "" {
|
|
return r.OutputText
|
|
}
|
|
for _, item := range r.Output {
|
|
for _, content := range item.Content {
|
|
if strings.TrimSpace(content.Text) != "" {
|
|
return content.Text
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func resolveSmokeBaseURL(cmd *cobra.Command) (string, error) {
|
|
if smokeBaseURL != "" {
|
|
return strings.TrimSuffix(smokeBaseURL, "/"), nil
|
|
}
|
|
cfg, _, err := LoadEdgeConfig(cmd)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to load edge config to resolve base URL (pass --base-url to override): %w", err)
|
|
}
|
|
listen := cfg.OpenAI.Listen
|
|
if listen == "" {
|
|
listen = "127.0.0.1:18081"
|
|
}
|
|
host, port, err := net.SplitHostPort(listen)
|
|
if err != nil {
|
|
return "http://127.0.0.1:18081", nil
|
|
}
|
|
if host == "0.0.0.0" || host == "" {
|
|
host = "127.0.0.1"
|
|
}
|
|
return "http://" + net.JoinHostPort(host, port), nil
|
|
}
|
|
|
|
func resolveSmokeAPIKey() (string, error) {
|
|
if strings.TrimSpace(smokeAPIKey) != "" {
|
|
return strings.TrimSpace(smokeAPIKey), nil
|
|
}
|
|
if strings.TrimSpace(smokeAPIKeyFile) != "" {
|
|
data, err := os.ReadFile(smokeAPIKeyFile)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read --api-key-file: %w", err)
|
|
}
|
|
return strings.TrimSpace(string(data)), nil
|
|
}
|
|
return strings.TrimSpace(os.Getenv("OPENAI_API_KEY")), nil
|
|
}
|
|
|
|
func smokeOpenAICmd() *cobra.Command {
|
|
c := &cobra.Command{
|
|
Use: "openai",
|
|
Short: "Perform OpenAI-compatible endpoint smoke test",
|
|
Long: `Validate the IOP Edge OpenAI-compatible HTTP API surface.
|
|
It runs a three-step diagnostics check:
|
|
1. Checks GET /healthz (server status).
|
|
2. Checks GET /v1/models (advertised model compatibility).
|
|
3. Checks POST /v1/responses (non-streaming Responses API request processing).
|
|
|
|
If --base-url is not provided, it auto-discovers the OpenAI URL from the effective config's listen address.`,
|
|
Example: ` iop-edge smoke openai --model gemma4:26b
|
|
iop-edge smoke openai --model gemma4:26b --base-url http://127.0.0.1:18081 --prompt "say hello" --timeout 15s`,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
baseURL, err := resolveSmokeBaseURL(cmd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
timeoutDur := 10 * time.Second
|
|
if smokeTimeout != "" {
|
|
d, err := time.ParseDuration(smokeTimeout)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid timeout duration: %w", err)
|
|
}
|
|
timeoutDur = d
|
|
}
|
|
|
|
client := &http.Client{
|
|
Timeout: timeoutDur,
|
|
}
|
|
apiKey, err := resolveSmokeAPIKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
doRequest := func(req *http.Request) (*http.Response, error) {
|
|
if apiKey != "" {
|
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
|
}
|
|
return client.Do(req)
|
|
}
|
|
|
|
fmt.Fprintf(cmd.OutOrStdout(), "IOP Edge OpenAI Smoke Test\n")
|
|
fmt.Fprintf(cmd.OutOrStdout(), "========================================\n")
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Target Endpoint: %s\n", baseURL)
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Target Model: %s\n", smokeModel)
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Timeout: %s\n\n", timeoutDur)
|
|
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Step 1: Checking /healthz ... ")
|
|
healthURL := baseURL + "/healthz"
|
|
req, err := http.NewRequest(http.MethodGet, healthURL, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("build GET %s: %w", healthURL, err)
|
|
}
|
|
resp, err := doRequest(req)
|
|
if err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("GET %s failed: %w (make sure the edge server is running)", healthURL, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("GET %s returned non-200 status %d: %s", healthURL, resp.StatusCode, string(body))
|
|
}
|
|
var healthData map[string]interface{}
|
|
if err := json.NewDecoder(resp.Body).Decode(&healthData); err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("failed to parse /healthz JSON response: %w", err)
|
|
}
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[OK]")
|
|
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Step 2: Checking /v1/models ... ")
|
|
modelsURL := baseURL + "/v1/models"
|
|
req, err = http.NewRequest(http.MethodGet, modelsURL, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("build GET %s: %w", modelsURL, err)
|
|
}
|
|
resp, err = doRequest(req)
|
|
if err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("GET %s failed: %w", modelsURL, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("GET %s returned non-200 status %d: %s", modelsURL, resp.StatusCode, string(body))
|
|
}
|
|
var modelsResp openAIModelsResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&modelsResp); err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("failed to parse /v1/models JSON response: %w", err)
|
|
}
|
|
modelIDs := make([]string, 0, len(modelsResp.Data))
|
|
modelMatched := false
|
|
for _, m := range modelsResp.Data {
|
|
modelIDs = append(modelIDs, m.ID)
|
|
if m.ID == smokeModel {
|
|
modelMatched = true
|
|
}
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "[OK] (found models: %s)\n", strings.Join(modelIDs, ", "))
|
|
if !modelMatched {
|
|
fmt.Fprintf(cmd.OutOrStdout(), "WARNING: Target model %q is not advertised by /v1/models (advertised: %s)\n", smokeModel, strings.Join(modelIDs, ", "))
|
|
}
|
|
|
|
prompt := smokePrompt
|
|
if prompt == "" {
|
|
prompt = "ping"
|
|
}
|
|
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Step 3: Checking /v1/responses ... ")
|
|
metadata := map[string]interface{}{
|
|
"request_id": "iop-edge-smoke",
|
|
"task_id": "smoke",
|
|
}
|
|
if smokeWorkspace != "" {
|
|
metadata["workspace"] = smokeWorkspace
|
|
}
|
|
|
|
responsesPayload := map[string]interface{}{
|
|
"model": smokeModel,
|
|
"input": prompt,
|
|
"stream": false,
|
|
"metadata": metadata,
|
|
"max_output_tokens": 32,
|
|
"temperature": 0,
|
|
}
|
|
payloadBytes, err := json.Marshal(responsesPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal responses request payload: %w", err)
|
|
}
|
|
|
|
responsesURL := baseURL + "/v1/responses"
|
|
req, err = http.NewRequest(http.MethodPost, responsesURL, bytes.NewReader(payloadBytes))
|
|
if err != nil {
|
|
return fmt.Errorf("build POST %s: %w", responsesURL, err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
resp, err = doRequest(req)
|
|
if err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("POST %s failed: %w (make sure active nodes are online)", responsesURL, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("POST %s returned non-200 status %d: %s", responsesURL, resp.StatusCode, string(body))
|
|
}
|
|
var responsesResp responsesResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&responsesResp); err != nil {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("failed to parse /v1/responses JSON response: %w", err)
|
|
}
|
|
outputText := responsesResp.text()
|
|
if strings.TrimSpace(outputText) == "" {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "[FAILED]")
|
|
return fmt.Errorf("POST %s returned empty response text", responsesURL)
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "[OK]\n")
|
|
fmt.Fprintf(cmd.OutOrStdout(), "Responses Output Text: %q\n\n", outputText)
|
|
|
|
if smokeExpectFile != "" {
|
|
if smokeWorkspace == "" {
|
|
return fmt.Errorf("--expect-file requires --workspace to be set")
|
|
}
|
|
expectFilePath := filepath.Join(smokeWorkspace, smokeExpectFile)
|
|
rel, err := filepath.Rel(smokeWorkspace, expectFilePath)
|
|
if err != nil || strings.HasPrefix(rel, "..") {
|
|
return fmt.Errorf("expect-file %q must be within the workspace", smokeExpectFile)
|
|
}
|
|
data, err := os.ReadFile(expectFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read expected file %q: %w", expectFilePath, err)
|
|
}
|
|
if smokeExpectContains != "" {
|
|
if !strings.Contains(string(data), smokeExpectContains) {
|
|
return fmt.Errorf("expected file %q does not contain expected substring %q", smokeExpectFile, smokeExpectContains)
|
|
}
|
|
}
|
|
}
|
|
|
|
fmt.Fprintf(cmd.OutOrStdout(), "IOP Edge OpenAI Smoke Test SUCCESS!\n")
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
c.Flags().StringVar(&smokeModel, "model", "", "model identifier to check (required)")
|
|
c.Flags().StringVar(&smokeBaseURL, "base-url", "", "override OpenAI base URL (e.g. http://127.0.0.1:18081)")
|
|
c.Flags().StringVar(&smokePrompt, "prompt", "ping", "prompt text to send")
|
|
c.Flags().StringVar(&smokeTimeout, "timeout", "10s", "client timeout duration")
|
|
c.Flags().StringVar(&smokeWorkspace, "workspace", "", "path to the temporary workspace directory")
|
|
c.Flags().StringVar(&smokeExpectFile, "expect-file", "", "relative path under workspace to check for existence after smoke")
|
|
c.Flags().StringVar(&smokeExpectContains, "expect-contains", "", "substring expected to be found in the expected file")
|
|
c.Flags().StringVar(&smokeAPIKey, "api-key", "", "bearer token for authenticated OpenAI-compatible endpoints (defaults to OPENAI_API_KEY)")
|
|
c.Flags().StringVar(&smokeAPIKeyFile, "api-key-file", "", "file containing bearer token for authenticated OpenAI-compatible endpoints")
|
|
c.MarkFlagRequired("model")
|
|
|
|
return c
|
|
}
|