fix(edge): Gemini 거부 출처를 구분한다
Edge 내부 오류가 provider HTTP 거부로 기록되지 않도록 실제 tunnel status에서만 관측하고, dev release가 stale tracking ref와 존재하지 않는 package root에 막히지 않게 한다.
This commit is contained in:
parent
28ed27a575
commit
0e17a9d9ec
7 changed files with 86 additions and 25 deletions
|
|
@ -1,6 +1,5 @@
|
|||
---
|
||||
name: dev-runtime-deploy
|
||||
version: 1.0.8
|
||||
description: dev 배포, dev-runtime 배포, Edge/Node dev 환경 배포 요청에서 dev commit count 기반 git-flow release를 만들고 clean sync, 전체 테스트, rebuild, 원격 배포, OpenAI-compatible capacity smoke, release finish와 tag push를 수행하는 절차
|
||||
---
|
||||
|
||||
|
|
@ -54,6 +53,8 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
|
|||
- [ ] 현재 구현의 completion 검증 대상은 legacy `/v1/completions`가 아니라 `/v1/chat/completions`이다. `/v1/completions`는 route가 구현되어 있을 때만 별도 검증한다.
|
||||
- [ ] 원격 runner에 git-flow가 설치되어 있고 `gitflow.branch.master=main`, `gitflow.branch.develop=dev`, `gitflow.prefix.release=release/`인지 확인한다.
|
||||
- [ ] shallow clone이 아니며 `origin/dev`, `origin/main`, tag를 fetch할 수 있는지 확인한다.
|
||||
- [ ] For a deployment that first merges a feature source into `dev`, run `git fetch --prune origin dev main <feature> --tags` and require the recorded exact feature SHA to match the fetched remote feature SHA. For a regular `dev` deployment, run `git fetch --prune origin dev main --tags`.
|
||||
- [ ] Use `git ls-remote --heads origin 'refs/heads/release/*'` as the authoritative remote release-head check. Treat stale local remote-tracking refs removed by prune as cleanup evidence, not as remote release blockers.
|
||||
- [ ] `origin/main`이 `origin/dev`의 ancestor인지 확인한다. 아니면 배포하지 않고 branch 정합화를 먼저 요구한다.
|
||||
- [ ] 계산된 tag가 이미 local이나 origin에 있으면 삭제·이동·덮어쓰기하지 않고 완료 상태 또는 partial finish 상태를 보고한다.
|
||||
- [ ] 계산된 `release/dev-<count>` 이외의 release branch가 있으면 중단한다. AVH git-flow는 다른 release가 진행 중이면 새 release start를 허용하지 않는다.
|
||||
|
|
@ -67,7 +68,8 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
|
|||
- 필수 정보가 없거나 서로 충돌하면 배포를 시작하지 말고 누락/충돌 항목을 보고한다.
|
||||
|
||||
2. **dev clean sync와 release version 확정**
|
||||
- 원격 runner checkout에서 `git fetch origin dev main --tags` 후 local `dev`를 `origin/dev`로 clean sync한다.
|
||||
- Run `git fetch --prune origin dev main --tags` in the remote runner checkout, then clean-sync local `dev` to `origin/dev`. When the deployment first merges a feature source, fetch the exact feature ref with `git fetch --prune origin dev main <feature> --tags`.
|
||||
- Determine active remote releases only from the authoritative output of `git ls-remote --heads origin 'refs/heads/release/*'`. Record changes to local `refs/remotes/origin/release/*` before and after prune only as stale-ref cleanup evidence.
|
||||
- dirty 파일은 보존 대상으로 보지 않는다. 배포 전 clean 상태를 만든다.
|
||||
- 기본 cleanup은 `git clean -fd`이다. `git clean -fdx`는 config, token, secret, runtime artifact까지 삭제할 수 있으므로 사용하지 않는다.
|
||||
- shallow clone이 아닌지, `HEAD`와 `origin/dev`가 같은 commit인지, `origin/main`이 `origin/dev`의 ancestor인지 확인한다.
|
||||
|
|
@ -92,7 +94,7 @@ dev-runtime provider pool을 `dev` 기준 git-flow release로 배포한다. `dev
|
|||
while IFS= read -r package; do
|
||||
go test -count=1 "$package"
|
||||
done < <(
|
||||
go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./cmd/... ./packages/go/... ./scripts/... |
|
||||
go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/... ./scripts/... |
|
||||
sed '/^iop\/packages\/go\/agenttask$/d'
|
||||
)
|
||||
```
|
||||
|
|
|
|||
|
|
@ -10,23 +10,31 @@ import (
|
|||
)
|
||||
|
||||
type geminiBridgeResponseWriter struct {
|
||||
target http.ResponseWriter
|
||||
header http.Header
|
||||
status int
|
||||
committed bool
|
||||
buffer bytes.Buffer
|
||||
stream *geminiBridgeStream
|
||||
target http.ResponseWriter
|
||||
header http.Header
|
||||
status int
|
||||
committed bool
|
||||
providerStatusObserved bool
|
||||
observeProviderStatus func(int)
|
||||
buffer bytes.Buffer
|
||||
stream *geminiBridgeStream
|
||||
}
|
||||
|
||||
func newGeminiBridgeResponseWriter(target http.ResponseWriter, model string) *geminiBridgeResponseWriter {
|
||||
func newGeminiBridgeResponseWriter(target http.ResponseWriter, model string, observeProviderStatus func(int)) *geminiBridgeResponseWriter {
|
||||
return &geminiBridgeResponseWriter{
|
||||
target: target, header: make(http.Header), stream: newGeminiBridgeStream(target, model),
|
||||
target: target, header: make(http.Header), observeProviderStatus: observeProviderStatus, stream: newGeminiBridgeStream(target, model),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *geminiBridgeResponseWriter) Header() http.Header { return w.header }
|
||||
|
||||
func (w *geminiBridgeResponseWriter) Status() int { return w.status }
|
||||
func (w *geminiBridgeResponseWriter) observeActualProviderHTTPStatus(status int) {
|
||||
if status < http.StatusBadRequest || w.providerStatusObserved || w.observeProviderStatus == nil {
|
||||
return
|
||||
}
|
||||
w.providerStatusObserved = true
|
||||
w.observeProviderStatus(status)
|
||||
}
|
||||
|
||||
func (w *geminiBridgeResponseWriter) WriteHeader(status int) {
|
||||
if w.status == 0 {
|
||||
|
|
|
|||
|
|
@ -83,11 +83,10 @@ func (s *Server) handleGeminiStreamGenerateContent(w http.ResponseWriter, r *htt
|
|||
internal.Header.Del("X-Goog-Api-Key")
|
||||
internal.Header.Set("Content-Type", "application/json")
|
||||
|
||||
bridge := newGeminiBridgeResponseWriter(w, callerModel)
|
||||
bridge := newGeminiBridgeResponseWriter(w, callerModel, func(status int) {
|
||||
s.observeGeminiRejection(geminiRejectionProviderHTTP, status)
|
||||
})
|
||||
s.handleChatCompletions(bridge, internal)
|
||||
if bridge.Status() >= http.StatusBadRequest {
|
||||
s.observeGeminiRejection(geminiRejectionProviderHTTP, bridge.Status())
|
||||
}
|
||||
bridge.Finish()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -321,10 +321,17 @@ func TestGeminiStreamBridgeEmitsBoundedToolOnce(t *testing.T) {
|
|||
func TestGeminiRejectionObservationIsClassificationOnly(t *testing.T) {
|
||||
core, observed := observer.New(zap.InfoLevel)
|
||||
secret := "secret-fixture-must-not-be-logged"
|
||||
srv := NewServer(config.EdgeOpenAIConf{BearerToken: "principal"}, &fakeRunService{}, zap.New(core))
|
||||
req := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":"`+secret+`"}`))
|
||||
req.Header.Set("X-Goog-Api-Key", "principal")
|
||||
srv.routes().ServeHTTP(httptest.NewRecorder(), req)
|
||||
|
||||
authServer := NewServer(config.EdgeOpenAIConf{BearerToken: "principal"}, &fakeRunService{}, zap.New(core))
|
||||
authReq := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"`+secret+`"}]}]}`))
|
||||
authServer.routes().ServeHTTP(httptest.NewRecorder(), authReq)
|
||||
|
||||
managedServer := managedCredentialMigrationServer(t, &providerFakeRunService{})
|
||||
managedServer.logger = zap.New(core)
|
||||
managedReq := httptest.NewRequest(http.MethodPost, "/gemini/public-route/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"hello"}]}]}`))
|
||||
managedReq.Header.Set("X-Goog-Api-Key", "managed-iop-token")
|
||||
managedReq.Header.Set(legacyProviderCredentialHeader, secret)
|
||||
managedServer.routes().ServeHTTP(httptest.NewRecorder(), managedReq)
|
||||
|
||||
candidate := anthropicTestCandidate(t, "openai")
|
||||
candidate.ActualModel = "served-chat"
|
||||
|
|
@ -340,14 +347,20 @@ func TestGeminiRejectionObservationIsClassificationOnly(t *testing.T) {
|
|||
providerReq.Header.Set("X-Goog-Api-Key", "principal")
|
||||
providerServer.routes().ServeHTTP(httptest.NewRecorder(), providerReq)
|
||||
|
||||
localFailureServer := NewServer(config.EdgeOpenAIConf{BearerToken: "principal", Adapter: "ollama", Target: "provider-model"}, &fakeRunService{submitErrAfter: 1}, zap.New(core))
|
||||
localFailureReq := httptest.NewRequest(http.MethodPost, "/gemini/r/v1beta/models/m:streamGenerateContent?alt=sse", strings.NewReader(`{"contents":[{"role":"user","parts":[{"text":"`+secret+`"}]}]}`))
|
||||
localFailureReq.Header.Set("X-Goog-Api-Key", "principal")
|
||||
localFailureServer.routes().ServeHTTP(httptest.NewRecorder(), localFailureReq)
|
||||
|
||||
entries := observed.FilterMessage(geminiRejectionLogMessage).All()
|
||||
if len(entries) != 2 {
|
||||
if len(entries) != 3 {
|
||||
t.Fatalf("rejection observations=%d logs=%+v", len(entries), observed.All())
|
||||
}
|
||||
wantClasses := []string{string(geminiRejectionPreIngress), string(geminiRejectionProviderHTTP)}
|
||||
wantClasses := []string{string(geminiRejectionPreIngress), string(geminiRejectionPreIngress), string(geminiRejectionProviderHTTP)}
|
||||
wantStatuses := []int64{http.StatusUnauthorized, http.StatusBadRequest, http.StatusBadRequest}
|
||||
for index, entry := range entries {
|
||||
context := entry.ContextMap()
|
||||
if len(context) != 4 || context["surface"] != "gemini" || context["bridge"] != "chat" || context["rejection_class"] != wantClasses[index] || context["http_status"] != int64(http.StatusBadRequest) {
|
||||
if len(context) != 4 || context["surface"] != "gemini" || context["bridge"] != "chat" || context["rejection_class"] != wantClasses[index] || context["http_status"] != wantStatuses[index] {
|
||||
t.Fatalf("unsafe or incomplete observation: %+v", context)
|
||||
}
|
||||
if strings.Contains(fmt.Sprint(context), secret) {
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ func (s *Server) writeCallerProviderCredentialRejection(w http.ResponseWriter, r
|
|||
return
|
||||
}
|
||||
if isGeminiRequest(r) {
|
||||
writeGeminiError(w, http.StatusBadRequest, "INVALID_ARGUMENT", message)
|
||||
s.writeGeminiPreIngressError(w, http.StatusBadRequest, "INVALID_ARGUMENT", message)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", message)
|
||||
|
|
@ -69,7 +69,7 @@ func (s *Server) writeAuthenticationFailure(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
if isGeminiRequest(r) {
|
||||
writeGeminiError(w, http.StatusUnauthorized, "UNAUTHENTICATED", "authentication failed")
|
||||
s.writeGeminiPreIngressError(w, http.StatusUnauthorized, "UNAUTHENTICATED", "authentication failed")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "unauthorized")
|
||||
|
|
|
|||
|
|
@ -17,6 +17,19 @@ type openAIRecoveryAdmissionAwareSink interface {
|
|||
setRecoveryAdmissionState(*openAIRecoveryAdmissionState)
|
||||
}
|
||||
|
||||
type actualProviderHTTPStatusObserver interface {
|
||||
observeActualProviderHTTPStatus(int)
|
||||
}
|
||||
|
||||
func notifyActualProviderHTTPStatus(w http.ResponseWriter, status int) {
|
||||
if status < http.StatusBadRequest {
|
||||
return
|
||||
}
|
||||
if observer, ok := w.(actualProviderHTTPStatusObserver); ok {
|
||||
observer.observeActualProviderHTTPStatus(status)
|
||||
}
|
||||
}
|
||||
|
||||
// bindOpenAIRecoveryAdmissionState gives a release sink the sanitized result of
|
||||
// recovery re-admission. It never exposes the raw dispatcher error to Core,
|
||||
// observations, or the caller.
|
||||
|
|
@ -303,6 +316,7 @@ func (s *openAITunnelReleaseSink) CommitResponseStart(ctx context.Context, rs st
|
|||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
notifyActualProviderHTTPStatus(s.w, status)
|
||||
s.w.WriteHeader(status)
|
||||
s.wroteHeader = true
|
||||
if s.flusher != nil {
|
||||
|
|
@ -406,6 +420,7 @@ func (s *openAITunnelReleaseSink) CommitTerminal(ctx context.Context, tr streamg
|
|||
if status == 0 {
|
||||
status = http.StatusBadGateway
|
||||
}
|
||||
notifyActualProviderHTTPStatus(s.w, status)
|
||||
s.w.WriteHeader(status)
|
||||
s.wroteHeader = true
|
||||
if len(response.body) > 0 {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ if str(_REPO_ROOT) not in sys.path:
|
|||
|
||||
_SKILL_DIR = _REPO_ROOT / "agent-ops" / "skills" / "project" / "iop-agent-comparison-benchmark"
|
||||
_SKILL_FILE = _SKILL_DIR / "SKILL.md"
|
||||
_DEPLOY_SKILL_FILE = (
|
||||
_REPO_ROOT / "agent-ops" / "skills" / "project" / "dev-runtime-deploy" / "SKILL.md"
|
||||
)
|
||||
_RULES_FILE = _REPO_ROOT / "agent-ops" / "rules" / "project" / "rules.md"
|
||||
_CLI_SCRIPT = _REPO_ROOT / "scripts" / "agent_comparison_benchmark.py"
|
||||
|
||||
|
|
@ -463,6 +466,27 @@ class BenchmarkSkillContractTest(unittest.TestCase):
|
|||
content = _SKILL_FILE.read_text(encoding="utf-8")
|
||||
self.assertRegex(content, r"description: .+", re.MULTILINE)
|
||||
|
||||
def test_dev_runtime_deploy_contract_is_authoritative_and_runnable(self) -> None:
|
||||
content = _DEPLOY_SKILL_FILE.read_text(encoding="utf-8")
|
||||
frontmatter = content.split("---", 2)[1]
|
||||
keys = [
|
||||
line.partition(":")[0]
|
||||
for line in frontmatter.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
self.assertEqual(keys, ["name", "description"])
|
||||
self.assertIn("git fetch --prune origin dev main <feature> --tags", content)
|
||||
self.assertIn(
|
||||
"git ls-remote --heads origin 'refs/heads/release/*'",
|
||||
content,
|
||||
)
|
||||
self.assertIn("stale local remote-tracking ref", content)
|
||||
self.assertIn(
|
||||
"go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/... ./scripts/...",
|
||||
content,
|
||||
)
|
||||
self.assertNotIn("./cmd/...", content)
|
||||
|
||||
def test_required_sections_present(self) -> None:
|
||||
content = _SKILL_FILE.read_text(encoding="utf-8")
|
||||
for section in (
|
||||
|
|
|
|||
Loading…
Reference in a new issue