toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/agent-test/dev/inventory.yaml
toki 0239ae9063 refactor: archive openai-compatible-provider-passthrough files and update configs 
- Move openai-compatible-provider-passthrough contract sync milestone and SDD to archive
- Update agent-spec for openai-compatible surface and edge-node-execution
- Update routing policy model orchestration phase
- Sync agent-test/dev/inventory.yaml
- Update .env.example files
- Update configs/edge.yaml and docker-compose.yml
2026-07-14 18:20:48 +09:00

951 lines
39 KiB
YAML
Raw Blame History

test_env: dev
profile: dev-runtime-provider-pool
last_updated_at: "2026-07-13"
source:
remote_runner:
ssh: toki@toki-labs.com
repo_root: /Users/toki/agent-work/iop-dev
clean_sync:
- git fetch origin main
- git reset --hard origin/main
- git clean -fd
dirty_policy: discard
compose:
project_name: iop-dev-agent
network: iop-dev-agent-net
subnet: 10.89.1.0/24
env_file: .env.dev.example
runtime_root: /Users/toki/agent-work/iop-dev
services:
- postgres
- redis
- control-plane
- web
- prometheus
- grafana
host_ports:
web: 13001
control_plane_http: 18001
control_plane_client_ws: 19001
control_plane_edge_wire: 19002
edge_node_tcp_compose: 19003
postgres: 15401
redis: 16301
edge_metrics_native: 19101
control_plane_metrics: 19103
prometheus: 19111
grafana: 19121
observability:
prometheus_config: configs/prometheus/prometheus.dev.yml
control_plane_metrics_target: control-plane:9093
edge_metrics_target: host.docker.internal:19101
grafana_bind: 127.0.0.1
prometheus_bind: 127.0.0.1
edge:
id: edge-toki-labs-dev
config_path: build/dev-runtime/edge.yaml
control_plane_http: http://127.0.0.1:18001
control_plane_status_url: http://127.0.0.1:18001/edges/edge-toki-labs-dev/status
bootstrap_http_public: http://toki-labs.com:18082
openai_base_url_public: http://toki-labs.com:18083/v1
openai_base_url_runner: http://127.0.0.1:18083/v1
edge_node_tcp_public: toki-labs.com:18084
admin_addr_runner: 127.0.0.1:19093
build:
binaries:
edge: build/dev-runtime/bin/edge
node_macos: build/dev-runtime/bin/iop-node
node_linux_arm64: build/dev-runtime/bin/iop-node-linux-arm64
node_windows_amd64: build/dev-runtime/bin/iop-node-windows-amd64.exe
model:
alias: ornith:35b
provider_capacity_total: 7
provider_capacity_status: verified_with_control_plane_provider_snapshots_2026_07_12
context_window: 262144
default_max_tokens: 32768
min_max_tokens: 16384
default_thinking_token_budget: 1024
qwen3_6_reference:
alias: qwen3.6:35b
status: registered_mac_mlx_only_after_ornith_replacement
provider_capacity_total: 2
provider: mac-mlx-vllm
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
context_window: 262144
default_max_tokens: 32768
min_max_tokens: 32768
active_edge_model_group:
observed_at: "2026-07-12"
id: ornith:35b
display_name: Ornith 1.0 35B
status: active_iop_edge_group
openai_base_url_public: http://toki-labs.com:18083/v1
context_window: 262144
default_max_tokens: 32768
min_max_tokens: 16384
default_thinking_token_budget: 1024
provider_capacity_total: 7
qwen_group_update: qwen3.6:35b remains registered with mac-mlx-vllm only; gx10-vllm and onexplayer-lemonade were removed from the Qwen group and now serve this Ornith group.
providers:
gx10-vllm:
runtime_type: vllm
endpoint: http://192.168.0.91:8001/v1
served_model: ornith:35b
upstream_model: deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
quantization: fp8
dtype: bfloat16
capacity: 4
total_context_tokens: 1048576
long_context_capacity: 4
runtime_args:
max_model_len: 262144
max_num_seqs: 4
gpu_memory_utilization: 0.50
enable_prefix_caching: true
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
trust_remote_code: true
language_model_only: true
observed_kv:
available_kv_cache_memory_gib: 20.79
gpu_kv_cache_tokens: 1060912
max_concurrency_for_262144_token_requests: 4.05
onexplayer-lemonade:
runtime_type: lemonade
endpoint: http://192.168.0.59:13305/v1
served_model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
response_model: ornith-1.0-35b-Q5_K_M.gguf
upstream_model: LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1
upstream_url: https://huggingface.co/LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1
upstream_revision: c50d5d4407f70e43208dee836c66bb8a05c1be91
quantization: Q5_K_M
capacity: 3
total_context_tokens: 524288
long_context_capacity: 2
runtime_args:
ctx_size: 524288
llamacpp_backend: vulkan
llamacpp_args: "--spec-type none -cb -fa on -b 4096 -ub 1024 --kv-unified"
fixed_parallel_slots_removed: true
observed_total_slots: 4
observed_slot_n_ctx: 262144
smoke:
models_endpoint: passed
chat_completions_single:
endpoint: /v1/chat/completions
request_model: ornith:35b
prompt: "2+2? Answer only the number."
result: passed
content: "4"
elapsed_sec: 3.32
completion_tokens: 127
responses_single:
endpoint: /v1/responses
request_model: ornith:35b
prompt: "2+2? Answer only the number."
result: passed
output_text: "4"
elapsed_sec: 4.58
completion_tokens: 1
chat_completions_capacity:
endpoint: /v1/chat/completions
concurrent_requests: 8
success_count: 8
failure_count: 0
wall_sec: 63.97
peak_total_in_flight: 7
peak_total_queued: 2
provider_peaks:
gx10-vllm:
capacity: 4
max_in_flight: 4
max_queued: 1
onexplayer-lemonade:
capacity: 3
max_in_flight: 3
max_queued: 1
mac-mlx-vllm:
capacity: 2
max_in_flight: 0
max_queued: 0
direct_throughput_benchmarks:
observed_at: "2026-07-12"
benchmark_shape: direct provider OpenAI-compatible /v1/chat/completions, max_tokens=1024, temperature=0, same long-list prompt, completion_tokens divided by wall time
benchmark_scope_note: throughput-only benchmark; all measured requests filled the 1024-token output budget and ended with finish_reason=length, so this is not a final-answer quality smoke.
gx10-vllm:
endpoint: http://192.168.0.91:8001/v1
model: ornith:35b
engine: vllm
quantization: fp8
results:
concurrency_1:
success_count: 1
failure_count: 0
avg_per_call_tok_s: 38.80
aggregate_tok_s: 38.80
wall_sec: 26.39
total_completion_tokens: 1024
concurrency_2:
success_count: 2
failure_count: 0
avg_per_call_tok_s: 36.95
aggregate_tok_s: 73.87
wall_sec: 27.72
total_completion_tokens: 2048
concurrency_3:
success_count: 3
failure_count: 0
avg_per_call_tok_s: 28.24
aggregate_tok_s: 84.68
wall_sec: 36.28
total_completion_tokens: 3072
concurrency_4:
success_count: 4
failure_count: 0
avg_per_call_tok_s: 31.46
aggregate_tok_s: 125.77
wall_sec: 32.57
total_completion_tokens: 4096
onexplayer-lemonade:
endpoint: http://192.168.0.59:13305/v1
model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
response_model: ornith-1.0-35b-Q5_K_M.gguf
engine: lemonade_llamacpp_vulkan
quantization: Q5_K_M
results:
concurrency_1:
success_count: 1
failure_count: 0
avg_per_call_tok_s: 45.07
aggregate_tok_s: 45.07
wall_sec: 22.72
total_completion_tokens: 1024
concurrency_2:
success_count: 2
failure_count: 0
avg_per_call_tok_s: 34.34
aggregate_tok_s: 68.67
wall_sec: 29.82
total_completion_tokens: 2048
concurrency_3:
success_count: 3
failure_count: 0
avg_per_call_tok_s: 23.27
aggregate_tok_s: 69.81
wall_sec: 44.01
total_completion_tokens: 3072
reasoning_policy: bounded_thinking
agent_tooling_policy:
runtime_profile: qwen_specific
mac_vllm_mlx:
tool_call_parser: qwen
reasoning_parser: qwen3
default_chat_template_kwargs:
enable_thinking: true
gx10_vllm:
reasoning_parser: qwen3
default_chat_template_kwargs:
enable_thinking: true
tool_call_parser_status: validate tools[] non-stream and streaming responses before enabling this provider for Pi/Cline-style agent tool use
separation_note: do not copy dev-corp Gemma parser/template values such as tool_call_parser=gemma4, reasoning_parser=gemma4, or Gemma4 chat templates into Qwen providers
pi_agent_profile:
observed_at: "2026-07-12"
config_scope: host_local_pi_agent
config_dir: /config/.pi/agent
current_default_provider: iop
current_default_model: ornith:35b
current_default_thinking_level: high
current_default_api: openai-completions
current_default_base_url: http://toki-labs.com:18083/v1
current_default_auth_header: true
api_key_value_tracked: false
endpoint_policy: current default Pi route is IOP Edge provider iop with model ornith:35b. Direct providers were removed from /config/.pi/agent/models.json so Pi no longer bypasses Edge to GX10 vLLM, OneXPlayer Lemonade, or the stopped DiffusionGemma endpoint.
local_pi_providers_at_observation:
- iop
direct_providers_removed_at_observation:
- iop-gemma
- ornith-direct
- ornith-lemonade
iop_provider_models_at_observation:
- qwen3.6:35b
- ornith:35b
pi_model_parameters:
provider: iop
model_id: ornith:35b
display_name: IOP Ornith 1.0 35B
api: openai-completions
context_window: 262144
max_tokens: 32768
reasoning: true
thinking_level: high
expected_iop_thinking_token_budget: 1024
current_ornith_parameter_alignment:
model_alias: ornith:35b
pi_context_window: 262144
edge_context_window: 262144
pi_max_tokens: 32768
edge_model_group_registered: true
edge_group_min_max_tokens: 16384
gx10_edge_capacity: 4
onexplayer_edge_capacity: 3
reasoning_parser: qwen3
tool_call_parser: qwen3_xml
fp8_required_on_gx10: true
note: Official 4-bit GGUF was not retained for GX10 vLLM because semantic generation smoke failed; the working GX10 runtime uses the official FP8 Hugging Face release.
runtime_parameter_alignment:
alignment_policy: Pi model parameters must be kept in sync with the Edge model catalog and each provider runtime profile. Do not optimize or debug Pi behavior from the endpoint URL alone; verify the alias-to-served-model mapping, context window, max token budget, thinking/reasoning parser, chat template kwargs, and provider capacity/sequence limits together.
common_requirements:
model_alias: qwen3.6:35b
pi_context_window: 262144
edge_context_window: 262144
pi_max_tokens: 32768
edge_default_max_tokens: 32768
edge_min_max_tokens: 32768
pi_reasoning: true
edge_reasoning_policy: bounded_thinking
edge_default_thinking_token_budget: 1024
required_template_policy: Qwen providers keep default_chat_template_kwargs.enable_thinking=true
required_reasoning_parser: qwen3
engine_matrix:
gx10-vllm:
runtime_type: vllm
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
context_match: pi contextWindow 262144 maps to vLLM max_model_len 262144
output_budget_match: pi maxTokens 32768 maps to provider default_max_tokens 32768
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=qwen3, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
concurrency_match: provider capacity 4 maps to runtime max_num_seqs 4
tool_parser_profile: qwen3_xml
mac-mlx-vllm:
runtime_type: vllm-mlx
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
context_match: pi contextWindow 262144 maps to vLLM-MLX max_request_tokens 262144 and max_kv_size 262144
output_budget_match: pi maxTokens 32768 maps to provider default_max_tokens 32768
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=qwen3, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
concurrency_match: provider capacity 2 maps to runtime max_num_seqs 2
tool_parser_profile: qwen
onexplayer-lemonade:
runtime_type: lemonade
served_model: Qwen3.6-35B-A3B-MTP-GGUF
current_runtime_override_2026_07_12: OneXPlayer Lemonade was switched to Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M for direct Ornith validation; it is not a valid Qwen provider-pool member until Edge provider config is updated.
context_match: Qwen should use the same Lemonade unified KV profile when re-enabled; keep total ctx_size 524288, omit -np, and pass --kv-unified to avoid the prior fixed 174848-token slot partitioning.
output_budget_match: pi maxTokens 32768 maps to load default_max_tokens 32768
thinking_match: pi reasoning=true and thinking_level=high map to IOP bounded thinking budget 1024; Lemonade load keeps MTP disabled and tool_choice_auto_verified=true
concurrency_match: provider capacity remains capped at 3 for IOP admission; with unified KV, Lemonade auto slots may report 4 runtime slots, so Edge admission must remain the concurrency guard.
tool_parser_profile: lemonade_qwen_tool_choice_auto_verified
qwen_reenable_profile:
status: applicable_not_reloaded_while_ornith_q5_is_active
ctx_size: 524288
llamacpp_args: "--spec-type none -cb -fa on -b 4096 -ub 1024 --kv-unified"
removed_args: "-np 3"
expected_slot_n_ctx: 262144
expected_capacity: 3
expected_long_context_capacity: 2
validation_basis: Same Lemonade llama.cpp backend and current Ornith Q5 runtime validated --kv-unified with auto slots reporting n_ctx 262144; Qwen still needs smoke validation after it is reloaded.
separation_note: These alignment rules are for the dev Qwen3.6 provider pool only. Gemma4 and DiffusionGemma use separate provider/model profiles and must not inherit Qwen parser/template/context assumptions.
provider_passthrough:
default: pure_passthrough
selector: none; request model/provider capability determines passthrough vs normalized execution
verification: Chat Completions streaming probe returned HTTP 200 through the provider tunnel with provider body preserved and no IOP response envelope injected.
provider_pool_routing:
observed_at: "2026-07-09"
caller_shape: Pi-compatible OpenAI Chat Completions streaming requests to provider/model iop/qwen3.6:35b
concurrent_requests: 10
peak_total_in_flight: 9
peak_total_queued_snapshot_units: 3
final_recovery: in_flight_0_queued_0
provider_peaks:
gx10-vllm:
runtime_type: vllm
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
capacity: 4
max_in_flight: 4
max_queued: 1
mac-mlx-vllm:
runtime_type: vllm-mlx
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
capacity: 2
max_in_flight: 2
max_queued: 1
onexplayer-lemonade:
runtime_type: lemonade
served_model: Qwen3.6-35B-A3B-MTP-GGUF
capacity: 3
max_in_flight: 3
max_queued: 1
non_default_local_provider:
provider: iop-gemma
status: removed_from_pi_models_json_2026_07_12
api: openai-completions
base_url: http://192.168.0.91:8002/v1
served_model: nvidia/diffusiongemma-26B-A4B-it-NVFP4
reasoning: false
current_runtime_status: stopped_2026_07_12_replaced_by_ornith_direct_fp8
spark_ornith_direct_profile:
observed_at: "2026-07-12"
provider: ornith-direct
status: removed_from_pi_models_json_2026_07_12_runtime_still_active_as_gx10_iop_ornith_provider
family: spark_ornith
api: openai-completions
base_url: http://192.168.0.91:8001/v1
model_id: ornith:35b
display_name: Ornith 1.0 35B FP8 Direct vLLM
context_window: 262144
max_tokens: 32768
reasoning: true
endpoint_policy: Pi no longer connects directly to this GX10/Spark vLLM endpoint. The same runtime remains registered through IOP Edge under ornith:35b.
api_key_value_tracked: false
pi_current_default_at_observation:
default_provider: iop
default_model: ornith:35b
default_api: openai-completions
default_base_url: http://toki-labs.com:18083/v1
qwen_parameter_alignment:
pi_context_window: 262144
pi_max_tokens: 32768
context_match: matches the dev Qwen Pi contextWindow and vLLM max_model_len value
output_budget_match: matches the dev Qwen Pi maxTokens/default_max_tokens value
runtime_summary:
host: toki@192.168.0.91
container_name: iop-vllm-ornith35b-fp8
docker_image: vllm/vllm-openai:nightly-aarch64
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
vllm_image_tag_env: vllm/vllm-openai:nightly-a346d589f5932d4234bf5bf8718f10e26d187021
vllm_version: 0.23.1rc1.dev223+ga346d589f
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
vllm_build_url: https://buildkite.com/vllm/release-v2/builds/2923
torch_version: 2.11.0+cu130
transformers_version: 5.12.1
compressed_tensors_version: 0.17.0
flashinfer_python_version: 0.6.12
cuda_version_env: 13.0.2
gpu: NVIDIA GB10
nvidia_driver_version: 580.126.09
endpoint: http://192.168.0.91:8001/v1
hf_model: deepreinforce-ai/Ornith-1.0-35B-FP8
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
hf_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
hf_ref: main
revision_argument: not_set_uses_repo_default_main_at_container_start
tokenizer_revision_argument: not_set_uses_model_revision
hf_home: /models/.cache/huggingface
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
served_model_name: ornith:35b
openai_models_root: deepreinforce-ai/Ornith-1.0-35B-FP8
openai_models_max_model_len: 262144
quantization: fp8
vllm_quantization_backend: compressed-tensors
model_type: qwen3_5_moe
model_architecture: Qwen3_5MoeForConditionalGeneration
quantization_config_source: model config.json compressed-tensors W8A8 FP8, weight channel strategy, input activation token strategy, linear_attn/embed/lm_head/shared gates ignored
dtype: bfloat16
max_model_len: 262144
max_num_seqs: 4
max_num_batched_tokens: vllm_default_not_overridden
gpu_memory_utilization: 0.50
enable_prefix_caching: true
enable_chunked_prefill: true
async_scheduling: true
tensor_parallel_size: 1
pipeline_parallel_size: 1
data_parallel_size: 1
load_format: auto
download_dir: null
kv_cache_dtype: auto
enforce_eager: false
seed: 0
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
reasoning_parser_enable_in_reasoning: false
trust_remote_code: true
language_model_only: true
attention_backend: FLASH_ATTN
sampling_backend: FlashInfer top-p/top-k
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
fp8_moe_backend: TRITON
generation_config_override:
temperature: 1.0
top_k: 20
top_p: 0.95
eos_token_id:
- 248046
- 248044
bos_token_id: 248044
docker_runtime:
created_at: "2026-07-12T08:41:04Z"
entrypoint:
- vllm
- serve
container_port: 8000
host_port: 8001
network_mode: bridge
ipc_mode: private
shm_size_bytes: 67108864
restart_policy: "no"
gpu_device_request: all
bind_mounts:
/home/toki/Data/models: /models
/home/toki/Data/vllm/templates: /templates
model_load_memory_gib: 34.43
model_load_seconds: 196.929359
weight_download_seconds: not_observed_cache_hit_restart
cuda_graph_memory_estimated_gib: 0.13
cuda_graph_pool_memory_actual_gib: 0.07
available_kv_cache_memory_gib: 20.79
gpu_kv_cache_tokens: 1060912
max_concurrency_for_262144_token_requests: 4.05
startup_warnings:
- vLLM command does not pin --revision; the recorded hf_revision is the downloaded snapshot observed in cache.
- Prefix caching in Mamba cache align mode is experimental for this architecture.
- Default MoE config for NVIDIA_GB10 FP8 was not found; vLLM used default MoE config, which may affect performance.
- Model generation_config overrides vLLM defaults unless callers provide sampling params or vLLM is relaunched with --generation-config vllm.
official_4bit_gguf_status:
hf_model: deepreinforce-ai/Ornith-1.0-35B-GGUF
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-GGUF
hf_revision: c2e1703039380de4ce6820e97afd185682d3c16c
hf_ref: main
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c
file: ornith-1.0-35b-Q4_K_M.gguf
status: not_active
reason: current vLLM GGUF plugin path exposed /v1/models but failed semantic generation smoke; active working runtime uses the official FP8 vLLM model instead.
smoke:
english_final_answer: passed
korean_final_answer: passed
pi_direct_cli: passed
output_throughput:
method: completion_tokens divided by end-to-end request wall time, max_tokens=768
average_completion_tok_s: 29.8
run_completion_tok_s:
- 29.65
- 30.62
- 29.15
note: vLLM logs showed aggregate generation throughput around 50-61 tok/s with two running requests.
ornith_lemonade_direct_profile:
observed_at: "2026-07-12"
provider: ornith-lemonade
status: removed_from_pi_models_json_2026_07_12_runtime_still_active_as_onexplayer_iop_ornith_provider
family: spark_ornith
api: openai-completions
base_url: http://192.168.0.59:13305/v1
model_id: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
display_name: Ornith 1.0 35B Q5 Direct Lemonade
context_window: 262144
max_tokens: 32768
reasoning: true
config_file: /config/.pi/agent/models.json
default_profile_changed: true
pi_default_provider_after_add: iop
pi_default_model_after_add: ornith:35b
endpoint_policy: Pi no longer connects directly to the OneXPlayer Lemonade OpenAI-compatible endpoint. This runtime remains reachable through IOP Edge under ornith:35b.
api_key_value_tracked: false
runtime_alignment:
lemonade_runtime_model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
lemonade_response_model: ornith-1.0-35b-Q5_K_M.gguf
quantization: Q5_K_M
ctx_size: 524288
kv_policy: unified_kv_with_np_omitted
observed_slot_n_ctx: 262144
iop_capacity_guard: 3
long_context_capacity: 2
smoke:
endpoint_direct_chat_completions: passed
request_model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
response_model: ornith-1.0-35b-Q5_K_M.gguf
finish_reason: stop
content: "4"
completion_tokens: 2
separation_note: This host Pi default profile is the dev Qwen3.6 provider-pool profile. Keep dev-corp Gemma4 under agent-test/dev-corp/inventory.yaml and do not copy Gemma4 parser/template/runtime settings into this Qwen profile.
content_completion_policy:
enforce_min_max_tokens: true
reason: prevent reasoning-only completions from exhausting caller max_tokens before final content
dispatch_policy:
capacity_gate: providers at or above capacity are excluded from dispatch candidates
selection_order: lower in_flight level wins among providers with available capacity
same_in_flight_tiebreak: lower numeric priority first, then deterministic rotation
capacity_smoke:
endpoints:
- /v1/responses
- /v1/chat/completions
concurrent_requests: 9
expected_total_in_flight: 9
expected_min_queued: 0
capacity_plus_one_concurrent_requests: 10
capacity_plus_one_expected_total_in_flight: 9
capacity_plus_one_expected_min_queued: 1
extended_concurrent_requests: 13
extended_expected_min_queued: 4
prompt_policy: long_reasoning_allowed
exact_output_match: false
latest_provider_snapshot:
observed_at: "2026-07-12"
total_capacity: 9
all_nodes_connected: true
providers:
gx10-vllm:
capacity: 4
priority: 0
in_flight: 0
queued: 0
health: healthy
served_model: ornith:35b
mac-mlx-vllm:
capacity: 2
priority: 2
in_flight: 0
queued: 0
health: healthy
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
onexplayer-lemonade:
capacity: 3
priority: 1
in_flight: 0
queued: 0
health: healthy
served_model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
openai_tool_call_smoke:
observed_at: "2026-07-08"
request_shape: tools_plus_tool_choice_auto
direct_provider_expected:
http_status: 200
minimum_tool_calls: 1
direct_provider_result:
gx10-vllm:
http_status: 200
tool_calls: 1
mac-mlx-vllm:
http_status: 200
tool_calls: 1
onexplayer-lemonade:
http_status: 200
tool_calls: 1
edge_concurrency_result:
endpoint: /v1/chat/completions
concurrent_requests: 10
success_count: 10
failure_count: 0
tool_call_response_count: 10
finish_reason_tool_calls: 10
peak_total_in_flight: 9
peak_total_queued: 3
provider_result_counts:
gx10-vllm: 5
onexplayer-lemonade: 3
mac-mlx-vllm: 2
provider_peaks:
gx10-vllm:
max_in_flight: 4
max_queued: 1
onexplayer-lemonade:
max_in_flight: 3
max_queued: 1
mac-mlx-vllm:
max_in_flight: 2
max_queued: 1
final_content_smoke:
endpoint: /v1/chat/completions
concurrent_requests: 10
current_capacity_total: 9
expected_peak_total_in_flight: 9
expected_min_queued: 1
request_max_tokens: 3072
effective_min_max_tokens: 32768
include_reasoning: false
expected_success_count: 10
expected_finish_reason: stop
expected_final_marker: true
expected_empty_content: 0
expected_iop_notice: 0
previous_observed_at: "2026-07-04"
previous_observed_capacity_total: 10
previous_observed_result:
http_ok: 10
success_count: 10
failure_count: 0
finish_reason_stop: 10
marker_missing_count: 0
empty_content_count: 0
iop_notice_count: 0
think_tag_leak_count: 0
reasoning_present_count: 0
aggregate_completion_tok_s: 62.5
total_completion_tokens: 17933
batch_wall_sec: 286.908
avg_latency_sec: 175.074
peak_total_in_flight: 10
peak_total_queued: 0
provider_peaks:
gx10-vllm:
capacity: 4
max_in_flight: 4
max_queued: 0
mac-mlx-vllm:
capacity: 3
max_in_flight: 3
max_queued: 0
onexplayer-lemonade:
capacity: 3
max_in_flight: 3
max_queued: 0
nodes:
- id: mac-codex-node
alias: mac-codex
role: cli+mlx-provider
ssh: toki@toki-labs.com
workspace: /Users/toki/agent-work/iop-workspace/nomadcode
provider_pool_candidate: true
adapters:
- cli
- mac-mlx-vllm
providers:
- id: mac-mlx-vllm
type: vllm-mlx
endpoint: http://127.0.0.1:8002/v1
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
capacity: 2
priority: 2
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
# Runtime KV budget is 262144 (max_kv_size / max_request_tokens), so only
# one full 262144-window long request fits. long_context_capacity is the
# long-slot count, distinct from the normal capacity (2).
total_context_tokens: 262144
long_context_capacity: 1
runtime:
api_key_policy: local_bearer_from_edge_yaml
workdir: /Users/toki/agent-work/iop-mlx-vllm
python: .venv/bin/python
package_baseline:
vllm_mlx: 0.3.0
mlx: 0.31.2
mlx_lm: 0.31.3
model_cache: /Users/toki/agent-work/iop-mlx-vllm/hf-cache
pid_file: /Users/toki/agent-work/iop-mlx-vllm/vllm-mlx.pid
stdout_log: /Users/toki/agent-work/iop-mlx-vllm/logs/vllm-mlx.stdout.log
stderr_log: /Users/toki/agent-work/iop-mlx-vllm/logs/vllm-mlx.stderr.log
max_num_seqs: 2
max_kv_size: 262144
max_request_tokens: 262144
default_max_tokens: 32768
default_thinking_token_budget: 1024
paged_cache_block_size: 64
max_cache_blocks: 4096
total_kv_tokens: 262144
enable_auto_tool_choice: true
tool_call_parser: qwen
reasoning_parser: qwen3
default_chat_template_kwargs:
enable_thinking: true
- id: gx10-vllm-node
alias: gx10-vllm
role: vllm-provider
ssh: toki@192.168.0.91
workspace: /home/toki/iop-gx10-vllm
provider_pool_candidate: true
provider:
id: gx10-vllm
type: vllm
endpoint: http://192.168.0.91:8001/v1
served_model: ornith:35b
capacity: 4
priority: 0
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
# Ornith FP8 runtime reports 1,060,912 GPU KV tokens and 4.05x maximum
# concurrency for full 262144-token requests, so four long slots are allowed.
total_context_tokens: 1048576
long_context_capacity: 4
runtime:
container_name: iop-vllm-ornith35b-fp8
max_model_len: 262144
max_num_seqs: 4
gpu_memory_utilization: 0.50
default_max_tokens: 32768
default_thinking_token_budget: 1024
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
trust_remote_code: true
language_model_only: true
quantization: fp8
dtype: bfloat16
direct_providers:
- id: ornith-direct
family: spark_ornith
type: vllm
endpoint: http://192.168.0.91:8001/v1
served_model: ornith:35b
upstream_model: deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
status: direct_pi_provider_removed_runtime_shares_current_iop_ornith_gx10_provider
pi_model_parameters:
context_window: 262144
max_tokens: 32768
reasoning: true
runtime:
container_name: iop-vllm-ornith35b-fp8
docker_image: vllm/vllm-openai:nightly-aarch64
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
vllm_version: 0.23.1rc1.dev223+ga346d589f
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
port_mapping: 0.0.0.0:8001->8000
hf_home: /models/.cache/huggingface
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
quantization: fp8
vllm_quantization_backend: compressed-tensors
dtype: bfloat16
max_model_len: 262144
max_num_seqs: 4
max_num_batched_tokens: vllm_default_not_overridden
gpu_memory_utilization: 0.50
enable_prefix_caching: true
enable_chunked_prefill: true
async_scheduling: true
tensor_parallel_size: 1
pipeline_parallel_size: 1
data_parallel_size: 1
load_format: auto
download_dir: null
kv_cache_dtype: auto
enforce_eager: false
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
reasoning_parser_enable_in_reasoning: false
trust_remote_code: true
language_model_only: true
attention_backend: FLASH_ATTN
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
fp8_moe_backend: TRITON
generation_config_override:
temperature: 1.0
top_k: 20
top_p: 0.95
docker_runtime:
network_mode: bridge
ipc_mode: private
shm_size_bytes: 67108864
restart_policy: "no"
gpu_device_request: all
bind_mounts:
/home/toki/Data/models: /models
/home/toki/Data/vllm/templates: /templates
model_load_memory_gib: 34.43
available_kv_cache_memory_gib: 20.79
gpu_kv_cache_tokens: 1060912
max_concurrency_for_262144_token_requests: 4.05
average_completion_tok_s: 29.8
replacement_note: DiffusionGemma direct vLLM on this endpoint was stopped; Ornith is served from the official FP8 Hugging Face release, not the 4-bit GGUF build.
- id: onexplayer-lemonade-node
alias: onexplayer-lemonade
role: lemonade-provider
ssh: r0bin@192.168.0.59
ssh_origin: current_host
workspace: C:/Users/r0bin/iop-field
provider_pool_candidate: true
provider:
id: onexplayer-lemonade
type: lemonade
endpoint: http://192.168.0.59:13305/v1
served_model: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
served_model_response: ornith-1.0-35b-Q5_K_M.gguf
previous_served_model: Qwen3.6-35B-A3B-MTP-GGUF
status: active_ornith_q5_current_iop_ornith_provider_pool_member
qwen_reenable_profile:
status: not_loaded_ornith_q5_active
model_name: Qwen3.6-35B-A3B-MTP-GGUF
checkpoint: unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
prior_saved_recipe_options:
ctx_size: 524288
llamacpp_args: "--spec-type none -np 3 -cb -fa on -b 4096 -ub 1024"
partition_note: The prior -np 3 profile fixed slot windows at about 174848 tokens for ctx_size 524288.
recommended_recipe_options:
ctx_size: 524288
llamacpp_args: "--spec-type none -cb -fa on -b 4096 -ub 1024 --kv-unified"
removed_args: "-np 3"
expected_slot_n_ctx: 262144
expected_auto_slots: 4
iop_capacity_guard: 3
long_context_capacity: 2
validation_required_after_load: true
validation_basis: Same Lemonade llama.cpp backend and current Ornith Q5 runtime confirmed --kv-unified with omitted -np prevents fixed ctx_size/parallel partitioning.
capacity: 3
priority: 1
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
# Ornith Q5 direct runtime keeps ctx_size=524288 and removes fixed -np
# partitioning by using llama.cpp unified KV. Do NOT raise ctx_size above 524288.
total_context_tokens: 524288
long_context_capacity: 2
load:
endpoint: http://192.168.0.59:13305/v1/load
model_name: Ornith-1.0-35B-GGUF-llamacpp-tp1-Q5_K_M
checkpoint: LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1:Q5_K_M
hf_url: https://huggingface.co/LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1
hf_revision: c50d5d4407f70e43208dee836c66bb8a05c1be91
hf_ref: main
gguf_file: ornith-1.0-35b-Q5_K_M.gguf
gguf_file_size_bytes: 24729130848
cache_snapshot_path: C:/Users/r0bin/.cache/huggingface/hub/models--LordNeel--Ornith-1.0-35B-GGUF-llamacpp-tp1/snapshots/c50d5d4407f70e43208dee836c66bb8a05c1be91
backend: vulkan
ctx_size: 524288
llamacpp_args: "--spec-type none -cb -fa on -b 4096 -ub 1024 --kv-unified"
observed_process_args: "--ctx-size 524288 --port 8001 --jinja --context-shift --keep 16 --reasoning-format auto --no-webui --no-mmap -ngl 99 --kv-unified --spec-type none -b 4096 -cb -fa on -ub 1024"
observed_total_slots: 4
observed_total_ctx_size: 524288
observed_slot_n_ctx: 262144
context_per_slot_note: -np is intentionally omitted; llama.cpp auto slots with --kv-unified avoid the prior fixed 524288/3 partition and report 262144-token slot windows backed by shared KV.
residency:
observed_at: "2026-07-12"
process_resident_policy: keep llama-server loaded until explicit lemonade unload or service/process restart
observed_process_running: true
observed_is_sleeping: false
idle_sleep_seconds: not_set_llama_cpp_default_minus_1_disabled
mmap_policy: disabled_via_no_mmap
memory_lock_policy: not_enabled_no_mlock_argument
note: This keeps the model warm in the active llama-server process, but it is not an OS-level hard memory pin; Windows can still page memory under pressure because --mlock is not set.
save_options: true
mtp_runtime_policy: disabled_via_spec_type_none
default_max_tokens: 32768
default_thinking_token_budget: 1024
tool_choice_auto_verified: true
smoke:
observed_at: "2026-07-12"
qwen_unloaded: true
load_status: passed
english_final_answer:
max_tokens: 512
result: passed
content: 2+2 equals 4.
completion_tokens: 414
generation_tok_s: 31.94
korean_final_answer:
max_tokens: 1024
result: passed
content: 2 더하기 2는 4입니다.
completion_tokens: 384
tool_choice_auto:
result: passed
finish_reason: tool_calls
tool_calls: 1
function_name: get_weather
arguments: "{\"city\":\"Seoul\"}"
unified_kv_concurrency:
observed_at: "2026-07-12"
result: passed
concurrent_requests: 3
finish_reason_stop: 3
final_content: "4"
total_elapsed_sec: 6.55
completion_tokens:
- 149
- 88
- 85
low_output_budget_note: max_tokens 128 and one Korean 512-token probe ended in reasoning_content with empty final content; keep Pi/Edge maxTokens at 32768 or use at least 1024 for short smoke prompts.
windows_process_start: Win32_Process.Create
Reference in a new issue View git blame Copy permalink