605 lines
25 KiB
YAML
605 lines
25 KiB
YAML
test_env: dev
|
|
profile: dev-runtime-provider-pool
|
|
last_updated_at: "2026-07-12"
|
|
|
|
source:
|
|
remote_runner:
|
|
ssh: toki@toki-labs.com
|
|
repo_root: /Users/toki/agent-work/iop-dev
|
|
clean_sync:
|
|
- git fetch origin main
|
|
- git reset --hard origin/main
|
|
- git clean -fd
|
|
dirty_policy: discard
|
|
|
|
edge:
|
|
id: edge-toki-labs-dev
|
|
config_path: build/dev-runtime/edge.yaml
|
|
control_plane_http: http://127.0.0.1:18001
|
|
control_plane_status_url: http://127.0.0.1:18001/edges/edge-toki-labs-dev/status
|
|
bootstrap_http_public: http://toki-labs.com:18082
|
|
openai_base_url_public: http://toki-labs.com:18083/v1
|
|
openai_base_url_runner: http://127.0.0.1:18083/v1
|
|
edge_node_tcp_public: toki-labs.com:18084
|
|
admin_addr_runner: 127.0.0.1:19093
|
|
|
|
build:
|
|
binaries:
|
|
edge: build/dev-runtime/bin/edge
|
|
node_macos: build/dev-runtime/bin/iop-node
|
|
node_linux_arm64: build/dev-runtime/bin/iop-node-linux-arm64
|
|
node_windows_amd64: build/dev-runtime/bin/iop-node-windows-amd64.exe
|
|
|
|
model:
|
|
alias: qwen3.6:35b
|
|
provider_capacity_total: 9
|
|
provider_capacity_status: verified_with_control_plane_provider_snapshots_2026_07_08
|
|
context_window: 262144
|
|
default_max_tokens: 32768
|
|
min_max_tokens: 32768
|
|
default_thinking_token_budget: 1024
|
|
reasoning_policy: bounded_thinking
|
|
agent_tooling_policy:
|
|
runtime_profile: qwen_specific
|
|
mac_vllm_mlx:
|
|
tool_call_parser: qwen
|
|
reasoning_parser: qwen3
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
gx10_vllm:
|
|
reasoning_parser: qwen3
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
tool_call_parser_status: validate tools[] non-stream and streaming responses before enabling this provider for Pi/Cline-style agent tool use
|
|
separation_note: do not copy dev-corp Gemma parser/template values such as tool_call_parser=gemma4, reasoning_parser=gemma4, or Gemma4 chat templates into Qwen providers
|
|
pi_agent_profile:
|
|
observed_at: "2026-07-09"
|
|
config_scope: host_local_pi_agent
|
|
config_dir: /config/.pi/agent
|
|
current_default_provider: iop
|
|
current_default_model: qwen3.6:35b
|
|
current_default_thinking_level: high
|
|
current_default_api: openai-completions
|
|
current_default_base_url: http://toki-labs.com:18083/v1
|
|
current_default_auth_header: true
|
|
api_key_value_tracked: false
|
|
endpoint_policy: current_default_base_url is the observed route for this host Pi profile, but optimization compatibility is determined by the Qwen3.6 model alias, context/output limits, reasoning policy, and provider runtime settings rather than by the endpoint literal alone.
|
|
pi_model_parameters:
|
|
provider: iop
|
|
model_id: qwen3.6:35b
|
|
display_name: IOP Qwen 3.6 35B
|
|
api: openai-completions
|
|
context_window: 262144
|
|
max_tokens: 32768
|
|
reasoning: true
|
|
thinking_level: high
|
|
expected_iop_thinking_token_budget: 1024
|
|
runtime_parameter_alignment:
|
|
alignment_policy: Pi model parameters must be kept in sync with the Edge model catalog and each provider runtime profile. Do not optimize or debug Pi behavior from the endpoint URL alone; verify the alias-to-served-model mapping, context window, max token budget, thinking/reasoning parser, chat template kwargs, and provider capacity/sequence limits together.
|
|
common_requirements:
|
|
model_alias: qwen3.6:35b
|
|
pi_context_window: 262144
|
|
edge_context_window: 262144
|
|
pi_max_tokens: 32768
|
|
edge_default_max_tokens: 32768
|
|
edge_min_max_tokens: 32768
|
|
pi_reasoning: true
|
|
edge_reasoning_policy: bounded_thinking
|
|
edge_default_thinking_token_budget: 1024
|
|
required_template_policy: Qwen providers keep default_chat_template_kwargs.enable_thinking=true
|
|
required_reasoning_parser: qwen3
|
|
engine_matrix:
|
|
gx10-vllm:
|
|
runtime_type: vllm
|
|
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
|
|
context_match: pi contextWindow 262144 maps to vLLM max_model_len 262144
|
|
output_budget_match: pi maxTokens 32768 maps to provider default_max_tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=qwen3, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
|
|
concurrency_match: provider capacity 4 maps to runtime max_num_seqs 4
|
|
tool_parser_profile: qwen3_xml
|
|
mac-mlx-vllm:
|
|
runtime_type: vllm-mlx
|
|
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
|
|
context_match: pi contextWindow 262144 maps to vLLM-MLX max_request_tokens 262144 and max_kv_size 262144
|
|
output_budget_match: pi maxTokens 32768 maps to provider default_max_tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=qwen3, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
|
|
concurrency_match: provider capacity 2 maps to runtime max_num_seqs 2
|
|
tool_parser_profile: qwen
|
|
onexplayer-lemonade:
|
|
runtime_type: lemonade
|
|
served_model: Qwen3.6-35B-A3B-MTP-GGUF
|
|
context_match: pi contextWindow 262144 is admitted within Lemonade ctx_size 524288 and current slot n_ctx 174848 x 3; long_context_capacity remains 2
|
|
output_budget_match: pi maxTokens 32768 maps to load default_max_tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to IOP bounded thinking budget 1024; Lemonade load keeps MTP disabled and tool_choice_auto_verified=true
|
|
concurrency_match: provider capacity 3 maps to Lemonade -np 3
|
|
tool_parser_profile: lemonade_qwen_tool_choice_auto_verified
|
|
separation_note: These alignment rules are for the dev Qwen3.6 provider pool only. Gemma4 and DiffusionGemma use separate provider/model profiles and must not inherit Qwen parser/template/context assumptions.
|
|
response_mode:
|
|
default: pure_passthrough
|
|
selector: metadata.iop_response_mode omitted or passthrough
|
|
verification: Chat Completions streaming probe returned HTTP 200 with no X-IOP-Response-Mode header and no iop.sideband marker; the same endpoint with metadata.iop_response_mode=passthrough+sideband returned X-IOP-Response-Mode=passthrough+sideband and iop.sideband, confirming mode separation.
|
|
provider_pool_routing:
|
|
observed_at: "2026-07-09"
|
|
caller_shape: Pi-compatible OpenAI Chat Completions streaming requests to provider/model iop/qwen3.6:35b
|
|
concurrent_requests: 10
|
|
peak_total_in_flight: 9
|
|
peak_total_queued_snapshot_units: 3
|
|
final_recovery: in_flight_0_queued_0
|
|
provider_peaks:
|
|
gx10-vllm:
|
|
runtime_type: vllm
|
|
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
|
|
capacity: 4
|
|
max_in_flight: 4
|
|
max_queued: 1
|
|
mac-mlx-vllm:
|
|
runtime_type: vllm-mlx
|
|
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
|
|
capacity: 2
|
|
max_in_flight: 2
|
|
max_queued: 1
|
|
onexplayer-lemonade:
|
|
runtime_type: lemonade
|
|
served_model: Qwen3.6-35B-A3B-MTP-GGUF
|
|
capacity: 3
|
|
max_in_flight: 3
|
|
max_queued: 1
|
|
non_default_local_provider:
|
|
provider: iop-gemma
|
|
status: separate_direct_provider_not_part_of_dev_qwen_provider_pool
|
|
api: openai-completions
|
|
base_url: http://192.168.0.91:8002/v1
|
|
served_model: nvidia/diffusiongemma-26B-A4B-it-NVFP4
|
|
reasoning: false
|
|
current_runtime_status: stopped_2026_07_12_replaced_by_ornith_direct_fp8
|
|
spark_ornith_direct_profile:
|
|
observed_at: "2026-07-12"
|
|
provider: ornith-direct
|
|
status: active_direct_pi_provider_not_part_of_iop_provider_pool
|
|
family: spark_ornith
|
|
api: openai-completions
|
|
base_url: http://192.168.0.91:8002/v1
|
|
model_id: ornith:35b
|
|
display_name: Ornith 1.0 35B FP8 Direct vLLM
|
|
context_window: 262144
|
|
max_tokens: 32768
|
|
reasoning: true
|
|
endpoint_policy: Pi connects directly to the GX10/Spark vLLM OpenAI-compatible endpoint; this profile bypasses IOP Edge routing and does not participate in the dev Qwen provider pool.
|
|
api_key_value_tracked: false
|
|
pi_current_default_at_observation:
|
|
default_provider: ornith-direct
|
|
default_model: ornith:35b
|
|
default_api: openai-completions
|
|
default_base_url: http://192.168.0.91:8002/v1
|
|
qwen_parameter_alignment:
|
|
pi_context_window: 262144
|
|
pi_max_tokens: 32768
|
|
context_match: matches the dev Qwen Pi contextWindow and vLLM max_model_len value
|
|
output_budget_match: matches the dev Qwen Pi maxTokens/default_max_tokens value
|
|
runtime_summary:
|
|
host: toki@192.168.0.91
|
|
container_name: iop-vllm-ornith35b-fp8
|
|
docker_image: vllm/vllm-openai:nightly-aarch64
|
|
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
|
|
vllm_image_tag_env: vllm/vllm-openai:nightly-a346d589f5932d4234bf5bf8718f10e26d187021
|
|
vllm_version: 0.23.1rc1.dev223+ga346d589f
|
|
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
|
|
vllm_build_url: https://buildkite.com/vllm/release-v2/builds/2923
|
|
torch_version: 2.11.0+cu130
|
|
transformers_version: 5.12.1
|
|
compressed_tensors_version: 0.17.0
|
|
flashinfer_python_version: 0.6.12
|
|
cuda_version_env: 13.0.2
|
|
gpu: NVIDIA GB10
|
|
nvidia_driver_version: 580.126.09
|
|
endpoint: http://192.168.0.91:8002/v1
|
|
hf_model: deepreinforce-ai/Ornith-1.0-35B-FP8
|
|
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
|
|
hf_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
|
|
hf_ref: main
|
|
revision_argument: not_set_uses_repo_default_main_at_container_start
|
|
tokenizer_revision_argument: not_set_uses_model_revision
|
|
hf_home: /models/.cache/huggingface
|
|
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
|
|
served_model_name: ornith:35b
|
|
openai_models_root: deepreinforce-ai/Ornith-1.0-35B-FP8
|
|
openai_models_max_model_len: 262144
|
|
quantization: fp8
|
|
vllm_quantization_backend: compressed-tensors
|
|
model_type: qwen3_5_moe
|
|
model_architecture: Qwen3_5MoeForConditionalGeneration
|
|
quantization_config_source: model config.json compressed-tensors W8A8 FP8, weight channel strategy, input activation token strategy, linear_attn/embed/lm_head/shared gates ignored
|
|
dtype: bfloat16
|
|
max_model_len: 262144
|
|
max_num_seqs: 2
|
|
max_num_batched_tokens: vllm_default_not_overridden
|
|
gpu_memory_utilization: 0.50
|
|
enable_prefix_caching: true
|
|
enable_chunked_prefill: true
|
|
async_scheduling: true
|
|
tensor_parallel_size: 1
|
|
pipeline_parallel_size: 1
|
|
data_parallel_size: 1
|
|
load_format: auto
|
|
download_dir: null
|
|
kv_cache_dtype: auto
|
|
enforce_eager: false
|
|
seed: 0
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: qwen3_xml
|
|
reasoning_parser: qwen3
|
|
reasoning_parser_enable_in_reasoning: false
|
|
trust_remote_code: true
|
|
language_model_only: true
|
|
attention_backend: FLASH_ATTN
|
|
sampling_backend: FlashInfer top-p/top-k
|
|
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
|
|
fp8_moe_backend: TRITON
|
|
generation_config_override:
|
|
temperature: 1.0
|
|
top_k: 20
|
|
top_p: 0.95
|
|
eos_token_id:
|
|
- 248046
|
|
- 248044
|
|
bos_token_id: 248044
|
|
docker_runtime:
|
|
created_at: "2026-07-12T04:40:26.013681328Z"
|
|
entrypoint:
|
|
- vllm
|
|
- serve
|
|
container_port: 8000
|
|
host_port: 8002
|
|
network_mode: bridge
|
|
ipc_mode: private
|
|
shm_size_bytes: 67108864
|
|
restart_policy: "no"
|
|
gpu_device_request: all
|
|
bind_mounts:
|
|
/home/toki/Data/models: /models
|
|
/home/toki/Data/vllm/templates: /templates
|
|
model_load_memory_gib: 34.43
|
|
model_load_seconds: 611.880026
|
|
weight_download_seconds: 407.868734
|
|
cuda_graph_memory_estimated_gib: 0.13
|
|
cuda_graph_pool_memory_actual_gib: 0.07
|
|
available_kv_cache_memory_gib: 21.33
|
|
gpu_kv_cache_tokens: 1087640
|
|
max_concurrency_for_262144_token_requests: 4.15
|
|
startup_warnings:
|
|
- vLLM command does not pin --revision; the recorded hf_revision is the downloaded snapshot observed in cache.
|
|
- Prefix caching in Mamba cache align mode is experimental for this architecture.
|
|
- Default MoE config for NVIDIA_GB10 FP8 was not found; vLLM used default MoE config, which may affect performance.
|
|
- Model generation_config overrides vLLM defaults unless callers provide sampling params or vLLM is relaunched with --generation-config vllm.
|
|
official_4bit_gguf_status:
|
|
hf_model: deepreinforce-ai/Ornith-1.0-35B-GGUF
|
|
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-GGUF
|
|
hf_revision: c2e1703039380de4ce6820e97afd185682d3c16c
|
|
hf_ref: main
|
|
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c
|
|
file: ornith-1.0-35b-Q4_K_M.gguf
|
|
status: not_active
|
|
reason: current vLLM GGUF plugin path exposed /v1/models but failed semantic generation smoke; active working runtime uses the official FP8 vLLM model instead.
|
|
smoke:
|
|
english_final_answer: passed
|
|
korean_final_answer: passed
|
|
pi_direct_cli: passed
|
|
output_throughput:
|
|
method: completion_tokens divided by end-to-end request wall time, max_tokens=768
|
|
average_completion_tok_s: 29.8
|
|
run_completion_tok_s:
|
|
- 29.65
|
|
- 30.62
|
|
- 29.15
|
|
note: vLLM logs showed aggregate generation throughput around 50-61 tok/s with two running requests.
|
|
separation_note: This host Pi default profile is the dev Qwen3.6 provider-pool profile. Keep dev-corp Gemma4 under agent-test/dev-corp/inventory.yaml and do not copy Gemma4 parser/template/runtime settings into this Qwen profile.
|
|
content_completion_policy:
|
|
enforce_min_max_tokens: true
|
|
reason: prevent reasoning-only completions from exhausting caller max_tokens before final content
|
|
dispatch_policy:
|
|
capacity_gate: providers at or above capacity are excluded from dispatch candidates
|
|
selection_order: lower in_flight level wins among providers with available capacity
|
|
same_in_flight_tiebreak: lower numeric priority first, then deterministic rotation
|
|
capacity_smoke:
|
|
endpoints:
|
|
- /v1/responses
|
|
- /v1/chat/completions
|
|
concurrent_requests: 9
|
|
expected_total_in_flight: 9
|
|
expected_min_queued: 0
|
|
capacity_plus_one_concurrent_requests: 10
|
|
capacity_plus_one_expected_total_in_flight: 9
|
|
capacity_plus_one_expected_min_queued: 1
|
|
extended_concurrent_requests: 13
|
|
extended_expected_min_queued: 4
|
|
prompt_policy: long_reasoning_allowed
|
|
exact_output_match: false
|
|
latest_provider_snapshot:
|
|
observed_at: "2026-07-08"
|
|
total_capacity: 9
|
|
all_nodes_connected: true
|
|
providers:
|
|
gx10-vllm:
|
|
capacity: 4
|
|
priority: 0
|
|
in_flight: 0
|
|
queued: 0
|
|
health: healthy
|
|
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
|
|
mac-mlx-vllm:
|
|
capacity: 2
|
|
priority: 2
|
|
in_flight: 0
|
|
queued: 0
|
|
health: healthy
|
|
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
|
|
onexplayer-lemonade:
|
|
capacity: 3
|
|
priority: 1
|
|
in_flight: 0
|
|
queued: 0
|
|
health: healthy
|
|
served_model: Qwen3.6-35B-A3B-MTP-GGUF
|
|
openai_tool_call_smoke:
|
|
observed_at: "2026-07-08"
|
|
request_shape: tools_plus_tool_choice_auto
|
|
direct_provider_expected:
|
|
http_status: 200
|
|
minimum_tool_calls: 1
|
|
direct_provider_result:
|
|
gx10-vllm:
|
|
http_status: 200
|
|
tool_calls: 1
|
|
mac-mlx-vllm:
|
|
http_status: 200
|
|
tool_calls: 1
|
|
onexplayer-lemonade:
|
|
http_status: 200
|
|
tool_calls: 1
|
|
edge_concurrency_result:
|
|
endpoint: /v1/chat/completions
|
|
concurrent_requests: 10
|
|
success_count: 10
|
|
failure_count: 0
|
|
tool_call_response_count: 10
|
|
finish_reason_tool_calls: 10
|
|
peak_total_in_flight: 9
|
|
peak_total_queued: 3
|
|
provider_result_counts:
|
|
gx10-vllm: 5
|
|
onexplayer-lemonade: 3
|
|
mac-mlx-vllm: 2
|
|
provider_peaks:
|
|
gx10-vllm:
|
|
max_in_flight: 4
|
|
max_queued: 1
|
|
onexplayer-lemonade:
|
|
max_in_flight: 3
|
|
max_queued: 1
|
|
mac-mlx-vllm:
|
|
max_in_flight: 2
|
|
max_queued: 1
|
|
final_content_smoke:
|
|
endpoint: /v1/chat/completions
|
|
concurrent_requests: 10
|
|
current_capacity_total: 9
|
|
expected_peak_total_in_flight: 9
|
|
expected_min_queued: 1
|
|
request_max_tokens: 3072
|
|
effective_min_max_tokens: 32768
|
|
include_reasoning: false
|
|
expected_success_count: 10
|
|
expected_finish_reason: stop
|
|
expected_final_marker: true
|
|
expected_empty_content: 0
|
|
expected_iop_notice: 0
|
|
previous_observed_at: "2026-07-04"
|
|
previous_observed_capacity_total: 10
|
|
previous_observed_result:
|
|
http_ok: 10
|
|
success_count: 10
|
|
failure_count: 0
|
|
finish_reason_stop: 10
|
|
marker_missing_count: 0
|
|
empty_content_count: 0
|
|
iop_notice_count: 0
|
|
think_tag_leak_count: 0
|
|
reasoning_present_count: 0
|
|
aggregate_completion_tok_s: 62.5
|
|
total_completion_tokens: 17933
|
|
batch_wall_sec: 286.908
|
|
avg_latency_sec: 175.074
|
|
peak_total_in_flight: 10
|
|
peak_total_queued: 0
|
|
provider_peaks:
|
|
gx10-vllm:
|
|
capacity: 4
|
|
max_in_flight: 4
|
|
max_queued: 0
|
|
mac-mlx-vllm:
|
|
capacity: 3
|
|
max_in_flight: 3
|
|
max_queued: 0
|
|
onexplayer-lemonade:
|
|
capacity: 3
|
|
max_in_flight: 3
|
|
max_queued: 0
|
|
|
|
nodes:
|
|
- id: mac-codex-node
|
|
alias: mac-codex
|
|
role: cli+mlx-provider
|
|
ssh: toki@toki-labs.com
|
|
workspace: /Users/toki/agent-work/iop-workspace/nomadcode
|
|
provider_pool_candidate: true
|
|
adapters:
|
|
- cli
|
|
- mac-mlx-vllm
|
|
providers:
|
|
- id: mac-mlx-vllm
|
|
type: vllm-mlx
|
|
endpoint: http://127.0.0.1:8002/v1
|
|
served_model: mlx-community/Qwen3.6-35B-A3B-4bit
|
|
capacity: 2
|
|
priority: 2
|
|
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
|
|
# Runtime KV budget is 262144 (max_kv_size / max_request_tokens), so only
|
|
# one full 262144-window long request fits. long_context_capacity is the
|
|
# long-slot count, distinct from the normal capacity (2).
|
|
total_context_tokens: 262144
|
|
long_context_capacity: 1
|
|
runtime:
|
|
api_key_policy: local_bearer_from_edge_yaml
|
|
workdir: /Users/toki/agent-work/iop-mlx-vllm
|
|
python: .venv/bin/python
|
|
package_baseline:
|
|
vllm_mlx: 0.3.0
|
|
mlx: 0.31.2
|
|
mlx_lm: 0.31.3
|
|
model_cache: /Users/toki/agent-work/iop-mlx-vllm/hf-cache
|
|
pid_file: /Users/toki/agent-work/iop-mlx-vllm/vllm-mlx.pid
|
|
stdout_log: /Users/toki/agent-work/iop-mlx-vllm/logs/vllm-mlx.stdout.log
|
|
stderr_log: /Users/toki/agent-work/iop-mlx-vllm/logs/vllm-mlx.stderr.log
|
|
max_num_seqs: 2
|
|
max_kv_size: 262144
|
|
max_request_tokens: 262144
|
|
default_max_tokens: 32768
|
|
default_thinking_token_budget: 1024
|
|
paged_cache_block_size: 64
|
|
max_cache_blocks: 4096
|
|
total_kv_tokens: 262144
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: qwen
|
|
reasoning_parser: qwen3
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
- id: gx10-vllm-node
|
|
alias: gx10-vllm
|
|
role: vllm-provider
|
|
ssh: toki@192.168.0.91
|
|
workspace: /home/toki/iop-gx10-vllm
|
|
provider_pool_candidate: true
|
|
provider:
|
|
id: gx10-vllm
|
|
type: vllm
|
|
endpoint: http://192.168.0.91:8001/v1
|
|
served_model: nvidia/Qwen3.6-35B-A3B-NVFP4
|
|
capacity: 4
|
|
priority: 0
|
|
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
|
|
# max_model_len is 262144; conservative long_context_capacity=1 until runtime
|
|
# evidence proves a larger concurrent long-context KV budget.
|
|
total_context_tokens: 262144
|
|
long_context_capacity: 1
|
|
runtime:
|
|
container_name: iop-vllm-qwen36-prev-20260704063128
|
|
max_model_len: 262144
|
|
max_num_seqs: 4
|
|
gpu_memory_utilization: 0.30
|
|
default_max_tokens: 32768
|
|
default_thinking_token_budget: 1024
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: qwen3_xml
|
|
reasoning_parser: qwen3
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
direct_providers:
|
|
- id: ornith-direct
|
|
family: spark_ornith
|
|
type: vllm
|
|
endpoint: http://192.168.0.91:8002/v1
|
|
served_model: ornith:35b
|
|
upstream_model: deepreinforce-ai/Ornith-1.0-35B-FP8
|
|
upstream_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
|
|
upstream_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
|
|
status: active_direct_pi_provider_not_iop_routed
|
|
pi_model_parameters:
|
|
context_window: 262144
|
|
max_tokens: 32768
|
|
reasoning: true
|
|
runtime:
|
|
container_name: iop-vllm-ornith35b-fp8
|
|
docker_image: vllm/vllm-openai:nightly-aarch64
|
|
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
|
|
vllm_version: 0.23.1rc1.dev223+ga346d589f
|
|
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
|
|
port_mapping: 0.0.0.0:8002->8000
|
|
hf_home: /models/.cache/huggingface
|
|
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
|
|
quantization: fp8
|
|
vllm_quantization_backend: compressed-tensors
|
|
dtype: bfloat16
|
|
max_model_len: 262144
|
|
max_num_seqs: 2
|
|
max_num_batched_tokens: vllm_default_not_overridden
|
|
gpu_memory_utilization: 0.50
|
|
enable_prefix_caching: true
|
|
enable_chunked_prefill: true
|
|
async_scheduling: true
|
|
tensor_parallel_size: 1
|
|
pipeline_parallel_size: 1
|
|
data_parallel_size: 1
|
|
load_format: auto
|
|
download_dir: null
|
|
kv_cache_dtype: auto
|
|
enforce_eager: false
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: qwen3_xml
|
|
reasoning_parser: qwen3
|
|
reasoning_parser_enable_in_reasoning: false
|
|
trust_remote_code: true
|
|
language_model_only: true
|
|
attention_backend: FLASH_ATTN
|
|
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
|
|
fp8_moe_backend: TRITON
|
|
generation_config_override:
|
|
temperature: 1.0
|
|
top_k: 20
|
|
top_p: 0.95
|
|
docker_runtime:
|
|
network_mode: bridge
|
|
ipc_mode: private
|
|
shm_size_bytes: 67108864
|
|
restart_policy: "no"
|
|
gpu_device_request: all
|
|
bind_mounts:
|
|
/home/toki/Data/models: /models
|
|
/home/toki/Data/vllm/templates: /templates
|
|
model_load_memory_gib: 34.43
|
|
available_kv_cache_memory_gib: 21.33
|
|
gpu_kv_cache_tokens: 1087640
|
|
max_concurrency_for_262144_token_requests: 4.15
|
|
average_completion_tok_s: 29.8
|
|
replacement_note: DiffusionGemma direct vLLM on this endpoint was stopped; Ornith is served from the official FP8 Hugging Face release, not the 4-bit GGUF build.
|
|
- id: onexplayer-lemonade-node
|
|
alias: onexplayer-lemonade
|
|
role: lemonade-provider
|
|
ssh: r0bin@192.168.0.59
|
|
ssh_origin: current_host
|
|
workspace: C:/Users/r0bin/iop-field
|
|
provider_pool_candidate: true
|
|
provider:
|
|
id: onexplayer-lemonade
|
|
type: lemonade
|
|
endpoint: http://192.168.0.59:13305/v1
|
|
served_model: Qwen3.6-35B-A3B-MTP-GGUF
|
|
capacity: 3
|
|
priority: 1
|
|
# Long-context admission policy (maps to edge.yaml nodes[].providers[]).
|
|
# SDD D04 fixes OneXPlayer dev baseline at ctx_size=524288, -np 3,
|
|
# long_context_capacity=2. Do NOT raise ctx_size above 524288.
|
|
total_context_tokens: 524288
|
|
long_context_capacity: 2
|
|
load:
|
|
endpoint: http://192.168.0.59:13305/v1/load
|
|
model_name: Qwen3.6-35B-A3B-MTP-GGUF
|
|
backend: vulkan
|
|
ctx_size: 524288
|
|
llamacpp_args: "--spec-type none -np 3 -cb -fa on -b 4096 -ub 1024"
|
|
observed_slot_n_ctx: 174848
|
|
save_options: true
|
|
mtp_runtime_policy: disabled
|
|
default_max_tokens: 32768
|
|
default_thinking_token_budget: 1024
|
|
tool_choice_auto_verified: true
|
|
windows_process_start: Win32_Process.Create
|