fix(edge): Gemini 거부 출처를 구분한다

Edge 내부 오류가 provider HTTP 거부로 기록되지 않도록 실제 tunnel status에서만 관측하고, dev release가 stale tracking ref와 존재하지 않는 package root에 막히지 않게 한다.
This commit is contained in:
toki 2026-08-13 10:34:33 +09:00
parent 28ed27a575
commit 0e17a9d9ec
7 changed files with 86 additions and 25 deletions

View file

@ -1,6 +1,5 @@
---
name: dev-runtime-deploy
version: 1.0.8
description: dev 배포, dev-runtime 배포, Edge/Node dev 환경 배포 요청에서 dev commit count 기반 git-flow release를 만들고 clean sync, 전체 테스트, rebuild, 원격 배포, OpenAI-compatible capacity smoke, release finish와 tag push를 수행하는 절차
---
@ -54,6 +53,8 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
- [ ] 현재 구현의 completion 검증 대상은 legacy `/v1/completions`가 아니라 `/v1/chat/completions`이다. `/v1/completions`는 route가 구현되어 있을 때만 별도 검증한다.
- [ ] 원격 runner에 git-flow가 설치되어 있고 `gitflow.branch.master=main`, `gitflow.branch.develop=dev`, `gitflow.prefix.release=release/`인지 확인한다.
- [ ] shallow clone이 아니며 `origin/dev`, `origin/main`, tag를 fetch할 수 있는지 확인한다.
- [ ] For a deployment that first merges a feature source into `dev`, run `git fetch --prune origin dev main <feature> --tags` and require the recorded exact feature SHA to match the fetched remote feature SHA. For a regular `dev` deployment, run `git fetch --prune origin dev main --tags`.
- [ ] Use `git ls-remote --heads origin 'refs/heads/release/*'` as the authoritative remote release-head check. Treat stale local remote-tracking refs removed by prune as cleanup evidence, not as remote release blockers.
- [ ] `origin/main``origin/dev`의 ancestor인지 확인한다. 아니면 배포하지 않고 branch 정합화를 먼저 요구한다.
- [ ] 계산된 tag가 이미 local이나 origin에 있으면 삭제·이동·덮어쓰기하지 않고 완료 상태 또는 partial finish 상태를 보고한다.
- [ ] 계산된 `release/dev-<count>` 이외의 release branch가 있으면 중단한다. AVH git-flow는 다른 release가 진행 중이면 새 release start를 허용하지 않는다.
@ -67,7 +68,8 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
- 필수 정보가 없거나 서로 충돌하면 배포를 시작하지 말고 누락/충돌 항목을 보고한다.
2. **dev clean sync와 release version 확정**
- 원격 runner checkout에서 `git fetch origin dev main --tags` 후 local `dev``origin/dev`로 clean sync한다.
- Run `git fetch --prune origin dev main --tags` in the remote runner checkout, then clean-sync local `dev` to `origin/dev`. When the deployment first merges a feature source, fetch the exact feature ref with `git fetch --prune origin dev main <feature> --tags`.
- Determine active remote releases only from the authoritative output of `git ls-remote --heads origin 'refs/heads/release/*'`. Record changes to local `refs/remotes/origin/release/*` before and after prune only as stale-ref cleanup evidence.
- dirty 파일은 보존 대상으로 보지 않는다. 배포 전 clean 상태를 만든다.
- 기본 cleanup은 `git clean -fd`이다. `git clean -fdx`는 config, token, secret, runtime artifact까지 삭제할 수 있으므로 사용하지 않는다.
- shallow clone이 아닌지, `HEAD``origin/dev`가 같은 commit인지, `origin/main``origin/dev`의 ancestor인지 확인한다.
@ -92,7 +94,7 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
while IFS= read -r package; do
go test -count=1 "$package"
done < <(
go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./cmd/... ./packages/go/... ./scripts/... |
go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/... ./scripts/... |
sed '/^iop\/packages\/go\/agenttask$/d'
)
```

View file

@ -10,23 +10,31 @@ import (
)
type geminiBridgeResponseWriter struct {
target http.ResponseWriter
header http.Header
status int
committed bool
buffer bytes.Buffer
stream *geminiBridgeStream
target http.ResponseWriter
header http.Header
status int
committed bool
providerStatusObserved bool
observeProviderStatus func(int)
buffer bytes.Buffer
stream *geminiBridgeStream
}
func newGeminiBridgeResponseWriter(target http.ResponseWriter, model string) *geminiBridgeResponseWriter {
func newGeminiBridgeResponseWriter(target http.ResponseWriter, model string, observeProviderStatus func(int)) *geminiBridgeResponseWriter {
return &geminiBridgeResponseWriter{
target: target, header: make(http.Header), stream: newGeminiBridgeStream(target, model),
target: target, header: make(http.Header), observeProviderStatus: observeProviderStatus, stream: newGeminiBridgeStream(target, model),
}
}
func (w *geminiBridgeResponseWriter) Header() http.Header { return w.header }
func (w *geminiBridgeResponseWriter) Status() int { return w.status }
func (w *geminiBridgeResponseWriter) observeActualProviderHTTPStatus(status int) {
if status < http.StatusBadRequest || w.providerStatusObserved || w.observeProviderStatus == nil {
return
}
w.providerStatusObserved = true
w.observeProviderStatus(status)
}
func (w *geminiBridgeResponseWriter) WriteHeader(status int) {
if w.status == 0 {

View file

@ -83,11 +83,10 @@ func (s *Server) handleGeminiStreamGenerateContent(w http.ResponseWriter, r *htt
internal.Header.Del("X-Goog-Api-Key")
internal.Header.Set("Content-Type", "application/json")
bridge := newGeminiBridgeResponseWriter(w, callerModel)
bridge := newGeminiBridgeResponseWriter(w, callerModel, func(status int) {
s.observeGeminiRejection(geminiRejectionProviderHTTP, status)
})
s.handleChatCompletions(bridge, internal)
if bridge.Status() >= http.StatusBadRequest {
s.observeGeminiRejection(geminiRejectionProviderHTTP, bridge.Status())
}
bridge.Finish()
}

View file

@ -321,10 +321,17 @@ func TestGeminiStreamBridgeEmitsBoundedToolOnce(t *testing.T) {
func TestGeminiRejectionObservationIsClassificationOnly(t *testing.T) {
core, observed := observer.New(zap.InfoLevel)
secret := "secret-fixture-must-not-be-logged"
srv := NewServer(config.EdgeOpenAIConf{BearerToken: "principal"}, &fakeRunService{}, zap.New(core))
req := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":"`+secret+`"}`))
req.Header.Set("X-Goog-Api-Key", "principal")
srv.routes().ServeHTTP(httptest.NewRecorder(), req)
authServer := NewServer(config.EdgeOpenAIConf{BearerToken: "principal"}, &fakeRunService{}, zap.New(core))
authReq := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"`+secret+`"}]}]}`))
authServer.routes().ServeHTTP(httptest.NewRecorder(), authReq)
managedServer := managedCredentialMigrationServer(t, &providerFakeRunService{})
managedServer.logger = zap.New(core)
managedReq := httptest.NewRequest(http.MethodPost, "/gemini/public-route/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"hello"}]}]}`))
managedReq.Header.Set("X-Goog-Api-Key", "managed-iop-token")
managedReq.Header.Set(legacyProviderCredentialHeader, secret)
managedServer.routes().ServeHTTP(httptest.NewRecorder(), managedReq)
candidate := anthropicTestCandidate(t, "openai")
candidate.ActualModel = "served-chat"
@ -340,14 +347,20 @@ func TestGeminiRejectionObservationIsClassificationOnly(t *testing.T) {
providerReq.Header.Set("X-Goog-Api-Key", "principal")
providerServer.routes().ServeHTTP(httptest.NewRecorder(), providerReq)
localFailureServer := NewServer(config.EdgeOpenAIConf{BearerToken: "principal", Adapter: "ollama", Target: "provider-model"}, &fakeRunService{submitErrAfter: 1}, zap.New(core))
localFailureReq := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"`+secret+`"}]}]}`))
localFailureReq.Header.Set("X-Goog-Api-Key", "principal")
localFailureServer.routes().ServeHTTP(httptest.NewRecorder(), localFailureReq)
entries := observed.FilterMessage(geminiRejectionLogMessage).All()
if len(entries) != 2 {
if len(entries) != 3 {
t.Fatalf("rejection observations=%d logs=%+v", len(entries), observed.All())
}
wantClasses := []string{string(geminiRejectionPreIngress), string(geminiRejectionProviderHTTP)}
wantClasses := []string{string(geminiRejectionPreIngress), string(geminiRejectionPreIngress), string(geminiRejectionProviderHTTP)}
wantStatuses := []int64{http.StatusUnauthorized, http.StatusBadRequest, http.StatusBadRequest}
for index, entry := range entries {
context := entry.ContextMap()
if len(context) != 4 || context["surface"] != "gemini" || context["bridge"] != "chat" || context["rejection_class"] != wantClasses[index] || context["http_status"] != int64(http.StatusBadRequest) {
if len(context) != 4 || context["surface"] != "gemini" || context["bridge"] != "chat" || context["rejection_class"] != wantClasses[index] || context["http_status"] != wantStatuses[index] {
t.Fatalf("unsafe or incomplete observation: %+v", context)
}
if strings.Contains(fmt.Sprint(context), secret) {

View file

@ -56,7 +56,7 @@ func (s *Server) writeCallerProviderCredentialRejection(w http.ResponseWriter, r
return
}
if isGeminiRequest(r) {
writeGeminiError(w, http.StatusBadRequest, "INVALID_ARGUMENT", message)
s.writeGeminiPreIngressError(w, http.StatusBadRequest, "INVALID_ARGUMENT", message)
return
}
writeError(w, http.StatusBadRequest, "invalid_request_error", message)
@ -69,7 +69,7 @@ func (s *Server) writeAuthenticationFailure(w http.ResponseWriter, r *http.Reque
return
}
if isGeminiRequest(r) {
writeGeminiError(w, http.StatusUnauthorized, "UNAUTHENTICATED", "authentication failed")
s.writeGeminiPreIngressError(w, http.StatusUnauthorized, "UNAUTHENTICATED", "authentication failed")
return
}
writeError(w, http.StatusUnauthorized, "unauthorized", "unauthorized")

View file

@ -17,6 +17,19 @@ type openAIRecoveryAdmissionAwareSink interface {
setRecoveryAdmissionState(*openAIRecoveryAdmissionState)
}
type actualProviderHTTPStatusObserver interface {
observeActualProviderHTTPStatus(int)
}
func notifyActualProviderHTTPStatus(w http.ResponseWriter, status int) {
if status < http.StatusBadRequest {
return
}
if observer, ok := w.(actualProviderHTTPStatusObserver); ok {
observer.observeActualProviderHTTPStatus(status)
}
}
// bindOpenAIRecoveryAdmissionState gives a release sink the sanitized result of
// recovery re-admission. It never exposes the raw dispatcher error to Core,
// observations, or the caller.
@ -303,6 +316,7 @@ func (s *openAITunnelReleaseSink) CommitResponseStart(ctx context.Context, rs st
if status == 0 {
status = http.StatusOK
}
notifyActualProviderHTTPStatus(s.w, status)
s.w.WriteHeader(status)
s.wroteHeader = true
if s.flusher != nil {
@ -406,6 +420,7 @@ func (s *openAITunnelReleaseSink) CommitTerminal(ctx context.Context, tr streamg
if status == 0 {
status = http.StatusBadGateway
}
notifyActualProviderHTTPStatus(s.w, status)
s.w.WriteHeader(status)
s.wroteHeader = true
if len(response.body) > 0 {

View file

@ -25,6 +25,9 @@ if str(_REPO_ROOT) not in sys.path:
_SKILL_DIR = _REPO_ROOT / "agent-ops" / "skills" / "project" / "iop-agent-comparison-benchmark"
_SKILL_FILE = _SKILL_DIR / "SKILL.md"
_DEPLOY_SKILL_FILE = (
_REPO_ROOT / "agent-ops" / "skills" / "project" / "dev-runtime-deploy" / "SKILL.md"
)
_RULES_FILE = _REPO_ROOT / "agent-ops" / "rules" / "project" / "rules.md"
_CLI_SCRIPT = _REPO_ROOT / "scripts" / "agent_comparison_benchmark.py"
@ -463,6 +466,27 @@ class BenchmarkSkillContractTest(unittest.TestCase):
content = _SKILL_FILE.read_text(encoding="utf-8")
self.assertRegex(content, r"description: .+", re.MULTILINE)
def test_dev_runtime_deploy_contract_is_authoritative_and_runnable(self) -> None:
content = _DEPLOY_SKILL_FILE.read_text(encoding="utf-8")
frontmatter = content.split("---", 2)[1]
keys = [
line.partition(":")[0]
for line in frontmatter.splitlines()
if line.strip()
]
self.assertEqual(keys, ["name", "description"])
self.assertIn("git fetch --prune origin dev main <feature> --tags", content)
self.assertIn(
"git ls-remote --heads origin 'refs/heads/release/*'",
content,
)
self.assertIn("stale local remote-tracking ref", content)
self.assertIn(
"go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/... ./scripts/...",
content,
)
self.assertNotIn("./cmd/...", content)
def test_required_sections_present(self) -> None:
content = _SKILL_FILE.read_text(encoding="utf-8")
for section in (