305 lines
12 KiB
Python
305 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Temp-HOME regression for agent-client/pi/install.sh.
|
|
|
|
Executes the real installer against an isolated temporary HOME and asserts
|
|
that Seulgivibe Codex remains api: openai-responses, that existing user
|
|
settings / API keys / dev-corp data and timeout semantics are preserved,
|
|
that idempotent re-runs are safe, and that no real user home or credential
|
|
is touched. Fake keys only — no network access required.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
from pathlib import Path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _installer_script() -> Path:
|
|
"""Return the absolute path to the real install.sh we are testing."""
|
|
return Path(__file__).resolve().parent / "install.sh"
|
|
|
|
|
|
def _run_installer(temp_home: Path) -> subprocess.CompletedProcess:
|
|
"""Invoke install.sh with HOME pointed at temp_home."""
|
|
env = os.environ.copy()
|
|
env["HOME"] = str(temp_home)
|
|
env.pop("PI_HOME", None) # ensure no override leaks in
|
|
result = subprocess.run(
|
|
["bash", str(_installer_script())],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _TempHomeMixin:
|
|
"""Common setUp / tearDown for temp-home tests."""
|
|
|
|
def setUp(self):
|
|
self.tmp_home = Path(tempfile.mkdtemp(prefix="pi-regression-"))
|
|
self.pi_dir = self.tmp_home / ".pi" / "agent"
|
|
self.pi_dir.mkdir(parents=True)
|
|
self.settings_path = self.pi_dir / "settings.json"
|
|
self.models_path = self.pi_dir / "models.json"
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.tmp_home, ignore_errors=True)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestPreservesOpenaiResponsesConsumptionAndUserSettings(_TempHomeMixin, unittest.TestCase):
|
|
"""Verify the consumer contract: api: openai-responses, defaults, keys, timeout."""
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.expected_dev_corp = {
|
|
"baseUrl": "https://dev.corp.example.com/v1",
|
|
"api": "openai-chat",
|
|
"apiKey": "sk-DEV-CORP-KEY",
|
|
"models": [{"id": "dev-model-1", "name": "Dev Corp Model"}],
|
|
}
|
|
|
|
# Seed representative existing user data.
|
|
# Custom defaultProvider/defaultModel are seeded to prove the installer
|
|
# does NOT overwrite them with setdefault.
|
|
self.settings_path.write_text(json.dumps({
|
|
"defaultProvider": "anthropic/claude-3-opus:high", # custom — must be preserved
|
|
"defaultModel": "claude-3-opus-20240401", # custom — must be preserved
|
|
"defaultThinkingLevel": "medium", # user preference — must be preserved
|
|
"hideThinkingBlock": True, # user preference
|
|
"httpIdleTimeoutMs": 300000, # user value — must be overridden to 0
|
|
"retry": {
|
|
"provider": {
|
|
"timeoutMs": 120000, # must be removed
|
|
"maxRetries": 3, # must be overridden to 0
|
|
"maxRetryDelayMs": 30000, # must be overridden
|
|
}
|
|
},
|
|
"extensions": [
|
|
"extensions/openai-sampling-parameters.ts", # must be removed
|
|
"extensions/custom-tool.ts", # must be preserved
|
|
],
|
|
"enabledModels": [
|
|
"seulgivibe-openai/gpt-5.1:medium", # legacy — must be removed
|
|
"seulgivibe-openai/gpt-5.5:xhigh", # legacy — must be removed
|
|
"anthropic/claude-3-opus:high", # third-party — must be preserved
|
|
],
|
|
"customTheme": "dark-plus", # arbitrary user data — must be preserved
|
|
}, indent=2))
|
|
|
|
self.models_path.write_text(json.dumps({
|
|
"providers": {
|
|
"seulgivibe-openai": {
|
|
"baseUrl": "https://old.example.com/openai/v1",
|
|
"api": "openai-chat",
|
|
"apiKey": "sk-OLD-USER-KEY", # must be preserved
|
|
"models": [
|
|
{"id": "gpt-5.1", "name": "Old GPT-5.1"},
|
|
],
|
|
},
|
|
"dev-corp": {
|
|
**self.expected_dev_corp,
|
|
},
|
|
},
|
|
}, indent=2))
|
|
|
|
def test_preserves_openai_responses_consumption_and_user_settings(self):
|
|
result = _run_installer(self.tmp_home)
|
|
self.assertEqual(result.returncode, 0, msg=f"stderr: {result.stderr}")
|
|
|
|
settings = json.loads(self.settings_path.read_text())
|
|
models = json.loads(self.models_path.read_text())
|
|
|
|
# 1) Seulgivibe Codex provider exists with api: openai-responses
|
|
codex = models["providers"]["seulgivibe-codex"]
|
|
self.assertEqual(codex["api"], "openai-responses")
|
|
self.assertEqual(codex["baseUrl"], "https://seulgivibe.lgudax.cool/openai/v1")
|
|
|
|
# 2) Custom defaultProvider is preserved (installer uses setdefault)
|
|
self.assertEqual(settings.get("defaultProvider"), "anthropic/claude-3-opus:high")
|
|
|
|
# 2b) Custom defaultModel is preserved (installer uses setdefault)
|
|
self.assertEqual(settings.get("defaultModel"), "claude-3-opus-20240401")
|
|
|
|
# 4) User's thinking level preference is preserved
|
|
self.assertEqual(settings["defaultThinkingLevel"], "medium")
|
|
|
|
# 5) User's hideThinkingBlock is preserved
|
|
self.assertEqual(settings["hideThinkingBlock"], True)
|
|
|
|
# 6) HTTP idle timeout is disabled (0)
|
|
self.assertEqual(settings["httpIdleTimeoutMs"], 0)
|
|
|
|
# 7) Provider timeout override is removed
|
|
provider_retry = settings["retry"]["provider"]
|
|
self.assertNotIn("timeoutMs", provider_retry)
|
|
|
|
# 8) Retry policy is set
|
|
self.assertEqual(provider_retry["maxRetries"], 0)
|
|
self.assertEqual(provider_retry["maxRetryDelayMs"], 60000)
|
|
|
|
# 9) Sampling extension is removed; other extensions preserved
|
|
self.assertNotIn("extensions/openai-sampling-parameters.ts", settings["extensions"])
|
|
self.assertIn("extensions/custom-tool.ts", settings["extensions"])
|
|
|
|
# 10) Legacy seulgivibe-openai entries removed from enabledModels
|
|
enabled = settings["enabledModels"]
|
|
for m in enabled:
|
|
self.assertFalse(m.startswith("seulgivibe-openai/"), f"legacy entry not removed: {m}")
|
|
# Third-party model preserved
|
|
self.assertIn("anthropic/claude-3-opus:high", enabled)
|
|
|
|
# 11) Custom user data preserved
|
|
self.assertEqual(settings["customTheme"], "dark-plus")
|
|
|
|
# 12) User API key preserved on codex provider
|
|
self.assertEqual(codex["apiKey"], "sk-OLD-USER-KEY")
|
|
|
|
# 13) Legacy seulgivibe-openai provider removed from models
|
|
self.assertNotIn("seulgivibe-openai", models["providers"])
|
|
|
|
# 14) dev-corp provider preserved — full document equality
|
|
dev = models["providers"]["dev-corp"]
|
|
self.assertEqual(dev, self.expected_dev_corp)
|
|
|
|
# 15) Codex models include expected entries
|
|
codex_model_ids = [m["id"] for m in codex.get("models", [])]
|
|
self.assertIn("gpt-5.1", codex_model_ids)
|
|
self.assertIn("gpt-5.5", codex_model_ids)
|
|
|
|
# 16) Legacy openai provider entry in enabledModels is removed
|
|
# (already checked above but be explicit about the contract)
|
|
legacy_in_enabled = [m for m in enabled if m.startswith("seulgivibe-openai/")]
|
|
self.assertEqual(legacy_in_enabled, [], "Legacy openai entries must be removed from enabledModels")
|
|
|
|
|
|
class TestIdempotentRerun(_TempHomeMixin, unittest.TestCase):
|
|
"""Re-running the installer must be safe and produce stable output."""
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.settings_path.write_text(json.dumps({
|
|
"defaultProvider": "seulgivibe-codex",
|
|
"defaultModel": "gpt-5.5",
|
|
"customSetting": "keep-me",
|
|
}, indent=2))
|
|
self.models_path.write_text(json.dumps({
|
|
"providers": {
|
|
"seulgivibe-codex": {
|
|
"baseUrl": "https://seulgivibe.lgudax.cool/openai/v1",
|
|
"api": "openai-responses",
|
|
"apiKey": "sk-EXISTING",
|
|
"models": [
|
|
{"id": "gpt-5.5", "name": "Seulgivibe Codex GPT-5.5", "input": ["text"]},
|
|
],
|
|
},
|
|
},
|
|
}, indent=2))
|
|
|
|
def test_is_idempotent(self):
|
|
# First run
|
|
r1 = _run_installer(self.tmp_home)
|
|
self.assertEqual(r1.returncode, 0, msg=f"stderr: {r1.stderr}")
|
|
|
|
# Capture complete parsed documents after first run.
|
|
settings_1 = json.loads(self.settings_path.read_text())
|
|
models_1 = json.loads(self.models_path.read_text())
|
|
|
|
# Second run (idempotent)
|
|
r2 = _run_installer(self.tmp_home)
|
|
self.assertEqual(r2.returncode, 0, msg=f"stderr: {r2.stderr}")
|
|
|
|
# Compare complete parsed documents across reruns.
|
|
settings_2 = json.loads(self.settings_path.read_text())
|
|
models_2 = json.loads(self.models_path.read_text())
|
|
self.assertEqual(settings_1, settings_2, "settings.json drifted across reruns")
|
|
self.assertEqual(models_1, models_2, "models.json drifted across reruns")
|
|
|
|
# Backup files were created (proves the installer ran and wrote)
|
|
bak_files = list(self.pi_dir.glob("*.bak-*"))
|
|
self.assertGreater(len(bak_files), 0, "Expected backup files from the second run")
|
|
|
|
|
|
class TestNoUserHomeTouch(_TempHomeMixin, unittest.TestCase):
|
|
"""Confirm the installer does not read or write outside the temp HOME."""
|
|
|
|
def test_installer_uses_only_test_home(self):
|
|
# Two distinct test-owned homes: decoy_home receives HOME override,
|
|
# installer_home is the actual target the installer writes to.
|
|
decoy_home = self.tmp_home / "decoy-home"
|
|
installer_home = self.tmp_home / "installer-home"
|
|
decoy_home.mkdir()
|
|
installer_home.mkdir()
|
|
|
|
# Seed only test-owned sentinel data in decoy_home.
|
|
decoy_settings = decoy_home / ".pi" / "agent" / "settings.json"
|
|
decoy_models = decoy_home / ".pi" / "agent" / "models.json"
|
|
sentinel = b"DECOY-SENTINEL-BYTES-DO-NOT-OVERWRITE"
|
|
decoy_settings.parent.mkdir(parents=True)
|
|
decoy_settings.write_bytes(sentinel)
|
|
# Leave decoy_models absent to prove absence state is preserved.
|
|
|
|
# Run installer with HOME pointing at decoy_home but installer writes
|
|
# to installer_home via the env HOME override.
|
|
with mock.patch.dict(os.environ, {"HOME": str(decoy_home)}):
|
|
result = _run_installer(installer_home)
|
|
self.assertEqual(result.returncode, 0, msg=f"stderr: {result.stderr}")
|
|
|
|
# Decoy home must remain untouched: sentinel bytes intact, absence preserved.
|
|
self.assertTrue(decoy_settings.exists(), "Decoy settings.json should still exist")
|
|
self.assertEqual(
|
|
decoy_settings.read_bytes(), sentinel,
|
|
"Decoy settings.json was modified by the installer!"
|
|
)
|
|
self.assertFalse(
|
|
decoy_models.exists(),
|
|
"Decoy models.json (absent) should remain absent",
|
|
)
|
|
|
|
|
|
class TestFakeKeysOnly(_TempHomeMixin, unittest.TestCase):
|
|
"""Ensure no real credentials appear in the temp HOME after install."""
|
|
|
|
def test_no_real_credentials(self):
|
|
# Seed with a fake key only.
|
|
self.models_path.write_text(json.dumps({
|
|
"providers": {
|
|
"seulgivibe-codex": {
|
|
"apiKey": "sk-FAKE-TEST-KEY",
|
|
},
|
|
},
|
|
}, indent=2))
|
|
|
|
result = _run_installer(self.tmp_home)
|
|
self.assertEqual(result.returncode, 0, msg=f"stderr: {result.stderr}")
|
|
|
|
models = json.loads(self.models_path.read_text())
|
|
codex_key = models["providers"]["seulgivibe-codex"].get("apiKey", "")
|
|
|
|
# Must not contain any real-looking key patterns.
|
|
self.assertNotIn("sk-live-", codex_key)
|
|
self.assertNotIn("sk-proj-", codex_key)
|
|
self.assertNotIn("sk-ant-", codex_key)
|
|
# Should retain the fake key we seeded.
|
|
self.assertEqual(codex_key, "sk-FAKE-TEST-KEY")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|