536 lines
26 KiB
YAML
536 lines
26 KiB
YAML
test_env: dev-corp
|
|
profile: dev-corp-provider-pool
|
|
last_updated_at: "2026-07-09"
|
|
|
|
source:
|
|
remote_runner:
|
|
ssh: fe@172.24.63.178
|
|
repo_root: /Users/fe/agent-work/iop-dev-corp
|
|
setup_required: true
|
|
setup_status: deployed
|
|
current_observation: /Users/fe/agent-work/iop-dev-corp checkout and build/dev-corp-runtime runtime were created on 2026-06-25, clean-synced/rebuilt at source ref 2d08d24 on 2026-07-06, then clean-synced/rebuilt/redeployed at source ref c2437aa on 2026-07-08 with 15-concurrency capacity smoke passed for /v1/responses and /v1/chat/completions; /Users/fe/iop-field remains unrelated legacy field state
|
|
clean_sync:
|
|
- git fetch origin main
|
|
- git reset --hard origin/main
|
|
- git clean -fd
|
|
dirty_policy: discard
|
|
|
|
edge:
|
|
id: dev-corp-edge
|
|
host: mac-mini
|
|
ssh: fe@172.24.63.178
|
|
os: macOS 26.2
|
|
hardware: Apple M2 Pro / 16GB
|
|
config_path: build/dev-corp-runtime/edge.yaml
|
|
control_plane_enabled_current_runtime: true
|
|
control_plane_http: http://127.0.0.1:18002
|
|
control_plane_status_url: http://127.0.0.1:18002/edges/dev-corp-edge/status
|
|
control_plane_client_ws_runner: ws://127.0.0.1:19004/client
|
|
control_plane_edge_wire_addr_runner: 127.0.0.1:19005
|
|
public_route_policy: default_dev_corp_external_route_uses_115_public_ip
|
|
edge_deploy_172_policy: explicit_user_request_only
|
|
public_route_note: 115.21.224.82 is the default dev-corp Edge deployment and smoke route; Edge deployment to 172.24.63.178 is prohibited by default and allowed only when the user explicitly requests the 172 route. 172.24.63.178 otherwise remains the mac-mini runner and node-only/internal route.
|
|
control_plane_http_public: http://115.21.224.82:18002
|
|
control_plane_client_ws_public: ws://115.21.224.82:19004/client
|
|
bootstrap_http_public: http://115.21.224.82:18085
|
|
bootstrap_http_node_internal: http://172.24.63.178:18085
|
|
openai_base_url_public: http://115.21.224.82:18086/v1
|
|
openai_base_url_node_internal: http://172.24.63.178:18086/v1
|
|
openai_base_url_runner: http://127.0.0.1:18086/v1
|
|
openai_api_key_required_current_runtime: true
|
|
openai_api_key_secret_path_remote: build/dev-corp-runtime/.secrets/openai_api_key
|
|
openai_api_key_value_tracked: false
|
|
edge_node_tcp_public: 115.21.224.82:18087
|
|
edge_node_tcp_node_internal: 172.24.63.178:18087
|
|
admin_addr_runner: 127.0.0.1:19094
|
|
|
|
build:
|
|
binaries:
|
|
control_plane: build/dev-corp-runtime/bin/control-plane
|
|
edge: build/dev-corp-runtime/bin/iop-edge
|
|
node_macos: build/dev-corp-runtime/bin/iop-node-darwin-arm64
|
|
node_linux_arm64: build/dev-corp-runtime/bin/iop-node-linux-arm64
|
|
node_windows_amd64: null
|
|
node_windows_amd64_note: not part of the default dev-corp provider pool
|
|
|
|
model:
|
|
alias: gemma4:26b
|
|
alias_status: candidate
|
|
alias_policy: expose one IOP model alias and map provider-specific served_model values per provider
|
|
provider_capacity_total: 13
|
|
provider_capacity_status: verified_with_control_plane_provider_snapshots_2026_07_08_15_concurrency
|
|
context_window_max: 262144
|
|
default_thinking_token_budget: 1024
|
|
reasoning_policy: bounded_thinking_enabled
|
|
think_policy:
|
|
default: enabled
|
|
strict_output_behavior: provider_pool_catalog_default_overrides_strict_disable
|
|
adapter_mapping: vllm_and_vllm_mlx_use_chat_template_kwargs_enable_thinking_true
|
|
pi_agent_profile:
|
|
observed_at: "2026-07-09"
|
|
config_scope: host_local_pi_agent
|
|
config_dir: ~/.pi/agent
|
|
install:
|
|
package: pi-coding-agent
|
|
version: 0.80.3
|
|
binary: /opt/homebrew/bin/pi
|
|
settings_file: ~/.pi/agent/settings.json
|
|
models_file: ~/.pi/agent/models.json
|
|
profile_namespace: dev_corp_gemma4_pi
|
|
current_default_provider: dev-corp
|
|
current_default_model: gemma4:26b
|
|
current_default_runtime: edge_openai_compatible_provider_pool
|
|
current_default_base_url: http://172.24.63.178:18086/v1
|
|
current_default_api: openai-completions
|
|
current_default_auth_header: true
|
|
current_default_thinking_level: high
|
|
hide_thinking_block: false
|
|
http_idle_timeout_ms: 300000
|
|
retry_provider_timeout_ms: 300000
|
|
retry_provider_max_retries: 0
|
|
retry_provider_max_retry_delay_ms: 60000
|
|
enabled_model_patterns:
|
|
- dev-corp-direct/**
|
|
- dev-corp-spark01/**
|
|
- dev-corp/*
|
|
api_key_value_tracked: false
|
|
route_policy: current host-local Pi default uses the 172.24.63.178 node-internal Edge route because this host can reach the mac-mini directly; this does not replace the default dev-corp public smoke/deploy route 115.21.224.82
|
|
pi_model_parameters:
|
|
provider: dev-corp
|
|
model_id: gemma4:26b
|
|
display_name: dev-corp gemma4:26b
|
|
api: openai-completions
|
|
context_window: 262144
|
|
max_tokens: 32768
|
|
reasoning: true
|
|
thinking_level: high
|
|
expected_iop_thinking_token_budget: 1024
|
|
compat:
|
|
supports_developer_role: false
|
|
supports_usage_in_streaming: false
|
|
max_tokens_field: max_tokens
|
|
supports_reasoning_effort: false
|
|
supports_strict_mode: false
|
|
supports_store: false
|
|
requires_tool_result_name: true
|
|
requires_assistant_after_tool_result: true
|
|
direct_provider_profiles:
|
|
mac_studio_vllm_mlx:
|
|
provider: dev-corp-direct
|
|
default_provider: false
|
|
base_url: http://172.24.63.178:8004/v1
|
|
served_model: mlx-community/gemma-4-26b-a4b-it-nvfp4
|
|
runtime: mac_studio_vllm_mlx
|
|
compat:
|
|
supports_strict_mode: true
|
|
thinking_format: chat-template
|
|
chat_template_kwargs:
|
|
enable_thinking: thinking.enabled
|
|
dgx_spark_01_vllm:
|
|
provider: dev-corp-spark01
|
|
default_provider: false
|
|
base_url: http://127.0.0.1:8002/v1
|
|
served_model: gemma-4-26B-A4B-it-NVFP4
|
|
runtime: dgx_spark_01_vllm
|
|
route_note: host-local direct or tunnel route only; keep as fallback and do not treat it as the Edge alias route
|
|
compat:
|
|
supports_strict_mode: true
|
|
thinking_format: chat-template
|
|
chat_template_kwargs:
|
|
enable_thinking: thinking.enabled
|
|
runtime_parameter_alignment:
|
|
alignment_policy: Pi model parameters must be kept in sync with the dev-corp Edge model alias, provider served_model values, context/output limits, Gemma parser/template policy, and provider capacity/sequence limits. Do not optimize or debug Pi behavior from the endpoint URL alone.
|
|
common_requirements:
|
|
model_alias: gemma4:26b
|
|
pi_context_window: 262144
|
|
edge_context_window: 262144
|
|
pi_max_tokens: 32768
|
|
provider_default_max_tokens: 32768
|
|
pi_reasoning: true
|
|
edge_reasoning_policy: bounded_thinking
|
|
edge_default_thinking_token_budget: 1024
|
|
required_template_policy: Gemma providers keep default_chat_template_kwargs.enable_thinking=true
|
|
required_tool_call_parser: gemma4
|
|
required_reasoning_parser: gemma4
|
|
engine_matrix:
|
|
dgx_spark_01_vllm:
|
|
runtime_type: vllm
|
|
served_model: gemma-4-26B-A4B-it-NVFP4
|
|
context_match: pi contextWindow 262144 maps to vLLM max_model_len 262144
|
|
output_budget_match: pi maxTokens 32768 maps to provider default max tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=gemma4, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
|
|
concurrency_match: provider capacity 4 maps to runtime max_num_seqs 4
|
|
tool_parser_profile: gemma4
|
|
dgx_spark_02_vllm:
|
|
runtime_type: vllm
|
|
served_model: gemma-4-26B-A4B-it-NVFP4
|
|
context_match: pi contextWindow 262144 maps to vLLM max_model_len 262144
|
|
output_budget_match: pi maxTokens 32768 maps to provider default max tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=gemma4, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
|
|
concurrency_match: provider capacity 4 maps to runtime max_num_seqs 4
|
|
tool_parser_profile: gemma4
|
|
mac_studio_vllm_mlx:
|
|
runtime_type: vllm-mlx
|
|
served_model: mlx-community/gemma-4-26b-a4b-it-nvfp4
|
|
context_match: pi contextWindow 262144 maps to vLLM-MLX max_request_tokens 262144 and max_kv_size 786432
|
|
output_budget_match: pi maxTokens 32768 maps to provider default max tokens 32768
|
|
thinking_match: pi reasoning=true and thinking_level=high map to reasoning_parser=gemma4, default_chat_template_kwargs.enable_thinking=true, and IOP thinking token budget 1024
|
|
concurrency_match: provider catalog capacity 5 maps to runtime headroom max_num_seqs 6
|
|
tool_parser_profile: gemma4
|
|
separation_note: This profile is for dev-corp Gemma4 Pi only. Keep dev Qwen3.6 Pi settings under agent-test/dev/inventory.yaml and do not copy Qwen parser/template/runtime assumptions into this Gemma profile or Gemma parser/template values into Qwen providers.
|
|
timeout_policy:
|
|
openai_timeout_sec: 1800
|
|
provider_queue_timeout_ms: 0
|
|
provider_queue_timeout_policy: disabled_no_iop_queue_timeout
|
|
backend_request_timeout_ms: 1800000
|
|
note: provider-pool long reasoning and long-context requests do not use an IOP queue timeout; backend/OpenAI run budgets remain 30 minutes
|
|
runtime_capacity_targets:
|
|
dgx_spark:
|
|
capacity: 4
|
|
context_window_max: 262144
|
|
kv_size_tokens_effective: 524288
|
|
kv_size_basis: requested minimum 262144x2; verify actual vLLM KV cache from startup log because gpu_memory_utilization controls available KV cache
|
|
fp4_moe_env: VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE=4194304
|
|
mac_studio:
|
|
capacity: 5
|
|
context_window_max: 262144
|
|
kv_size_tokens_effective: 786432
|
|
kv_size_basis: requested 262144x3 mapped to vLLM-MLX max_kv_size
|
|
capacity_smoke:
|
|
endpoints:
|
|
- /v1/responses
|
|
- /v1/chat/completions
|
|
concurrent_requests: 15
|
|
expected_total_in_flight: 13
|
|
expected_total_in_flight_status: verified_15_concurrency_2026_07_08
|
|
expected_min_queued: 2
|
|
previous_capacity_plus_one_concurrent_requests: 14
|
|
previous_capacity_plus_one_expected_min_queued: 1
|
|
prompt_policy: long_reasoning_allowed
|
|
think_policy: enabled_for_provider_pool
|
|
exact_output_match: false
|
|
latest_capacity_verification:
|
|
date: "2026-07-08"
|
|
source_ref: c2437aaedefbac4312d69dfd10aa017c2739e187
|
|
edge_runtime_patch: docs_15_concurrency_standard_redeploy
|
|
default_thinking_token_budget: 1024
|
|
control_plane_enabled: true
|
|
control_plane_status_url: http://127.0.0.1:18002/edges/dev-corp-edge/status
|
|
runtime:
|
|
runner_checkout: /Users/fe/agent-work/iop-dev-corp
|
|
remote_log: build/dev-corp-runtime/logs/capacity_15_20260708_155027.json
|
|
control_plane_pid: 1446
|
|
edge_pid: 1481
|
|
rebuild_time: "2026-07-08 15:48"
|
|
post_build_config_check: passed
|
|
post_build_refresh_dry_run: no_changes_detected
|
|
provider_snapshot_baseline:
|
|
total_capacity: 13
|
|
providers:
|
|
corp-dgx-spark-01-vllm:
|
|
capacity: 4
|
|
health: healthy
|
|
status: available
|
|
corp-dgx-spark-02-vllm:
|
|
capacity: 4
|
|
health: healthy
|
|
status: available
|
|
corp-mac-studio-mlx-vllm:
|
|
capacity: 5
|
|
health: healthy
|
|
status: available
|
|
responses:
|
|
concurrent_requests: 15
|
|
ok: 15
|
|
timeout_errors: 0
|
|
http_errors: 0
|
|
max_total_in_flight: 13
|
|
max_total_queued: 6
|
|
elapsed_sec: 22.563
|
|
provider_max:
|
|
corp-dgx-spark-01-vllm:
|
|
max_in_flight: 4
|
|
max_queued: 2
|
|
corp-dgx-spark-02-vllm:
|
|
max_in_flight: 4
|
|
max_queued: 2
|
|
corp-mac-studio-mlx-vllm:
|
|
max_in_flight: 5
|
|
max_queued: 2
|
|
final_recovery: in_flight_0_queued_0
|
|
chat_completions:
|
|
concurrent_requests: 15
|
|
ok: 15
|
|
timeout_errors: 0
|
|
http_errors: 0
|
|
max_total_in_flight: 13
|
|
max_total_queued: 6
|
|
elapsed_sec: 25.117
|
|
provider_max:
|
|
corp-dgx-spark-01-vllm:
|
|
max_in_flight: 4
|
|
max_queued: 2
|
|
corp-dgx-spark-02-vllm:
|
|
max_in_flight: 4
|
|
max_queued: 2
|
|
corp-mac-studio-mlx-vllm:
|
|
max_in_flight: 5
|
|
max_queued: 2
|
|
final_recovery: in_flight_0_queued_0
|
|
latest_runtime_update:
|
|
date: "2026-06-25"
|
|
reason: user requested dev-corp node capacity/context/KV alignment
|
|
requested_targets:
|
|
dgx_spark:
|
|
capacity: 4
|
|
context_window_max: 262144
|
|
kv_size: 262144x2
|
|
mac_studio:
|
|
capacity: 5
|
|
context_window_max: 262144
|
|
kv_size: 262144x3
|
|
applied_mapping:
|
|
dgx_spark: vLLM has no vLLM-MLX style max_kv_size flag; requested KV 262144x2 is validated from startup log KV cache size and max_model_len 262144
|
|
mac_studio: vLLM-MLX requested KV 262144x3 is represented as max_kv_size 786432 with max_request_tokens 262144
|
|
remediation_applied:
|
|
- DGX Spark vLLM FP4 MoE first profile required VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE=4194304 for max_num_batched_tokens 524288
|
|
- DGX Spark 01/02 Gemma4 agent serving now uses vLLM 0.24.0 with text-only/eager profile to keep native tool-call streaming stable for Pi/Cline-style agents
|
|
- DGX Spark 01 start script now prefixes /home/digitalcommerce_dgx_spark_01/vllm_env_0_24_0/bin
|
|
- Mac Studio vLLM-MLX pyexpat required DYLD_LIBRARY_PATH=/opt/homebrew/opt/expat/lib
|
|
verification:
|
|
last_rechecked_at: "2026-07-08"
|
|
mac_studio_health: passed
|
|
dgx01_health: passed
|
|
dgx02_health: passed
|
|
direct_health_observation:
|
|
dgx01: mac-mini http://192.168.2.2:8002 returned /health 200 and /v1/models exposed gemma-4-26B-A4B-it-NVFP4 after vLLM 0.24.0 text-only/eager restart with default_chat_template_kwargs.enable_thinking=true; direct API forced/auto/streaming tool-call smoke and local Pi TUI bash/git-push tool-call flows passed; startup log reported GPU KV cache size 1,453,705 tokens and 5.55x concurrency for 262,144-token requests
|
|
dgx02: node-local http://127.0.0.1:8004 and mac-mini http://192.168.2.4:8004 returned /health 200, /v1/models exposed gemma-4-26B-A4B-it-NVFP4 after Docker image vllm/vllm-openai:v0.24.0 text-only/eager restart with default_chat_template_kwargs.enable_thinking=true; direct API auto/streaming tool-call smoke passed; startup log reported GPU KV cache size 1,452,633 tokens and 5.54x concurrency for 262,144-token requests
|
|
mac_studio: node-local http://127.0.0.1:8004, mac-mini http://192.168.2.3:8004, and current-host http://172.24.63.178:8004 returned /health 200, /v1/models exposed mlx-community/gemma-4-26b-a4b-it-nvfp4 after vllm-mlx 0.4.0 restart with continuous batching and disable_prefix_cache; direct API non-stream/stream auto tool-call, streaming multi-turn tool-result final-answer, Pi -p tool-call, and Pi TUI initial-prompt tool-call server log completion passed
|
|
edge_openai_smoke: passed_with_control_plane_capacity_snapshot_2026_07_06
|
|
|
|
nodes:
|
|
- id: corp-dgx-spark-01-vllm-node
|
|
alias: corp-dgx-spark-01-vllm
|
|
role: vllm-provider
|
|
ssh: digitalcommerce_dgx_spark_01@192.168.2.2
|
|
ssh_origin: mac-mini
|
|
workspace: /home/digitalcommerce_dgx_spark_01
|
|
provider_pool_candidate: true
|
|
provider:
|
|
id: corp-dgx-spark-01-vllm
|
|
type: vllm
|
|
endpoint: http://192.168.2.2:8002/v1
|
|
edge_adapter_endpoint: http://127.0.0.1:8002/v1
|
|
health: http://192.168.2.2:8002/health
|
|
served_model: gemma-4-26B-A4B-it-NVFP4
|
|
capacity: 4
|
|
capacity_status: configured_health_passed_direct_smoke_and_edge_capacity_smoke
|
|
last_runtime_observation: upgraded on 2026-07-08 to vLLM 0.24.0 with text-only/eager Gemma4 agent profile and default_chat_template_kwargs.enable_thinking=true; mac-mini http://192.168.2.2:8002 returned /health 200 and /v1/models, direct API forced/auto/streaming tool-call smoke passed, and local Pi TUI via SSH tunnel completed bash and git push tool-call flows; startup log reported GPU KV cache size 1,453,705 tokens and 5.55x concurrency for 262,144-token requests
|
|
capacity_basis: dev-corp target baseline; vLLM 0.24.0 --max-num-seqs 4, --max-model-len 262144, --gpu-memory-utilization 0.40, text-only/eager profile; observed KV cache satisfies requested 262144x2 minimum
|
|
queue_timeout_ms: 0
|
|
request_timeout_ms: 1800000
|
|
edge_connectivity:
|
|
mode: reverse_ssh_tunnel
|
|
reason: current node.yaml uses tunnel-local Edge addr in the native provider-pool runtime
|
|
tunnel_pid_file: /Users/fe/agent-work/iop-dev-corp/build/dev-corp-runtime/node01-tunnel.pid
|
|
remote_forwards:
|
|
artifact: 127.0.0.1:28085 -> mac-mini 127.0.0.1:18085
|
|
edge_node_tcp: 127.0.0.1:28087 -> mac-mini 127.0.0.1:18087
|
|
runtime:
|
|
host: spark-0b30
|
|
os: Ubuntu 24.04.3 LTS / aarch64
|
|
hardware: NVIDIA DGX Spark / NVIDIA GB10 / 119GiB RAM
|
|
manager: tmux
|
|
tmux_session: vllm_server_8002
|
|
start_script: /home/digitalcommerce_dgx_spark_01/start_vllm_8002.sh
|
|
python: /home/digitalcommerce_dgx_spark_01/vllm_env_0_24_0/bin/python3
|
|
path_prefix_required: /home/digitalcommerce_dgx_spark_01/vllm_env_0_24_0/bin
|
|
path_prefix_reason: FlashInfer FP4 JIT invokes ninja from PATH during first compile
|
|
package_baseline:
|
|
vllm: 0.24.0
|
|
torch: 2.11.0+cu130
|
|
transformers: 5.13.0
|
|
flashinfer_python: 0.6.12
|
|
huggingface_hub: 1.22.0
|
|
args:
|
|
env:
|
|
PATH_prefix: /home/digitalcommerce_dgx_spark_01/vllm_env_0_24_0/bin
|
|
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "4194304"
|
|
host: 0.0.0.0
|
|
port: 8002
|
|
dtype: bfloat16
|
|
gpu_memory_utilization: 0.40
|
|
max_model_len: 262144
|
|
max_num_seqs: 4
|
|
enforce_eager: true
|
|
skip_mm_profiling: true
|
|
limit_mm_per_prompt:
|
|
image: 0
|
|
video: 0
|
|
mm_processor_cache_gb: 0
|
|
text_only_mode: true
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: gemma4
|
|
chat_template: /home/digitalcommerce_dgx_spark_01/tool_chat_template_gemma4.jinja
|
|
reasoning_parser: gemma4
|
|
max_num_batched_tokens: auto_2496
|
|
max_num_batched_tokens_note: vLLM raised the default from 2048 to 2496 for the Gemma4 prefix-LM/video path; it is a scheduler iteration budget, not the total KV cache size
|
|
context_window_max: 262144
|
|
kv_size_tokens_effective: 1453705
|
|
full_context_concurrency_observed: 5.55
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
|
|
- id: corp-dgx-spark-02-vllm-node
|
|
alias: corp-dgx-spark-02-vllm
|
|
role: vllm-provider
|
|
ssh: dplab@192.168.2.4
|
|
ssh_origin: mac-mini
|
|
workspace: /home/dplab
|
|
provider_pool_candidate: true
|
|
provider:
|
|
id: corp-dgx-spark-02-vllm
|
|
type: vllm
|
|
endpoint: http://192.168.2.4:8004/v1
|
|
edge_adapter_endpoint: http://127.0.0.1:8004/v1
|
|
health: http://192.168.2.4:8004/health
|
|
served_model: gemma-4-26B-A4B-it-NVFP4
|
|
capacity: 4
|
|
capacity_status: configured_health_passed_direct_smoke_and_edge_capacity_smoke
|
|
last_runtime_observation: upgraded on 2026-07-08 to Docker image vllm/vllm-openai:v0.24.0 with text-only/eager Gemma4 agent profile and default_chat_template_kwargs.enable_thinking=true; node-local and mac-mini /health passed, /v1/models exposed gemma-4-26B-A4B-it-NVFP4, and direct API auto plus streaming tool-call smoke passed; startup log reported GPU KV cache size 1,452,633 tokens and 5.54x concurrency for 262,144-token requests
|
|
capacity_basis: dev-corp target baseline; Docker publishes host 8004 to container 8004; vLLM 0.24.0 --max-num-seqs 4, --max-model-len 262144, --gpu-memory-utilization 0.40, text-only/eager profile; observed KV cache satisfies requested 262144x2 minimum
|
|
queue_timeout_ms: 0
|
|
request_timeout_ms: 1800000
|
|
edge_connectivity:
|
|
mode: reverse_ssh_tunnel
|
|
reason: node02 cannot reach mac-mini 172.24.63.178:18085/18086 directly from 192.168.2.4
|
|
tunnel_pid_file: /Users/fe/agent-work/iop-dev-corp/build/dev-corp-runtime/node02-tunnel.pid
|
|
remote_forwards:
|
|
artifact: 127.0.0.1:28085 -> mac-mini 127.0.0.1:18085
|
|
edge_node_tcp: 127.0.0.1:28087 -> mac-mini 127.0.0.1:18087
|
|
runtime:
|
|
host: spark-fb94
|
|
os: Ubuntu 24.04.3 LTS / aarch64
|
|
hardware: NVIDIA DGX Spark / NVIDIA GB10 / 119GiB RAM
|
|
manager: docker
|
|
container_name: vllm-gemma4
|
|
start_script: /home/dplab/start_vllm_gemma4_8004.sh
|
|
env_file: /home/dplab/.config/iop-dev-corp/vllm-gemma4.env
|
|
env_file_contains_secret: true
|
|
image: vllm/vllm-openai:v0.24.0
|
|
container_port: 8004
|
|
host_port: 8004
|
|
package_baseline:
|
|
vllm: 0.24.0
|
|
torch: 2.11.0+cu130
|
|
transformers: 5.12.1
|
|
flashinfer_python: 0.6.12
|
|
huggingface_hub: 1.21.0
|
|
args:
|
|
env:
|
|
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "4194304"
|
|
gpu_memory_utilization: 0.40
|
|
max_model_len: 262144
|
|
max_num_seqs: 4
|
|
enforce_eager: true
|
|
skip_mm_profiling: true
|
|
limit_mm_per_prompt:
|
|
image: 0
|
|
video: 0
|
|
mm_processor_cache_gb: 0
|
|
text_only_mode: true
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: gemma4
|
|
chat_template: /vllm-workspace/examples/tool_chat_template_gemma4.jinja
|
|
reasoning_parser: gemma4
|
|
max_num_batched_tokens: auto_2496
|
|
max_num_batched_tokens_note: vLLM raised the default from 2048 to 2496 for the Gemma4 prefix-LM/video path; it is a scheduler iteration budget, not the total KV cache size
|
|
context_window_max: 262144
|
|
kv_size_tokens_effective: 1452633
|
|
full_context_concurrency_observed: 5.54
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
|
|
- id: corp-mac-studio-mlx-vllm-node
|
|
alias: corp-mac-studio-mlx-vllm
|
|
role: vllm-mlx-provider
|
|
ssh: dc_dev@192.168.2.3
|
|
ssh_origin: mac-mini
|
|
workspace: /Users/dc_dev
|
|
provider_pool_candidate: true
|
|
provider:
|
|
id: corp-mac-studio-mlx-vllm
|
|
type: openai_compat
|
|
runtime_type: vllm-mlx
|
|
endpoint: http://192.168.2.3:8004/v1
|
|
edge_adapter_endpoint: http://127.0.0.1:8004/v1
|
|
health: http://192.168.2.3:8004/health
|
|
served_model: mlx-community/gemma-4-26b-a4b-it-nvfp4
|
|
capacity: 5
|
|
capacity_status: configured_health_passed_direct_smoke_and_edge_capacity_smoke
|
|
last_runtime_observation: screen vllm_mlx_8004 running vllm-mlx 0.4.0 with provider catalog capacity 5, continuous batching enabled, runtime headroom max_num_seqs 6, max_request_tokens 262144, max_kv_size 786432, disable_prefix_cache, and default_chat_template_kwargs.enable_thinking=true; node-local, mac-mini, and current-host /health passed, /v1/models exposed mlx-community/gemma-4-26b-a4b-it-nvfp4, direct API non-stream/stream auto tool-call and streaming multi-turn tool-result final-answer smoke passed, Pi -p tool-call passed, and Pi TUI initial-prompt tool-call completed on the server log
|
|
capacity_basis: provider catalog capacity 5; vllm-mlx 0.4.0 runtime headroom uses --max-num-seqs 6 with requested KV 262144x3 mapped to --max-kv-size 786432; continuous batching is required for this provider, while prefix cache is disabled for Gemma4 stability
|
|
working_checkpoint:
|
|
status: final_working_baseline_as_of_2026_07_08
|
|
do_not_change_with_spark_vllm: true
|
|
micro_tuning_policy: only in a separate experiment with an explicit rollback point; do not change Spark vLLM settings or Pi default provider/model as part of this tuning
|
|
required_runtime_flags:
|
|
- --continuous-batching
|
|
- --max-num-seqs 6
|
|
- --prefill-batch-size 6
|
|
- --completion-batch-size 6
|
|
- --chunked-prefill-tokens 1024
|
|
- --disable-prefix-cache
|
|
- --max-kv-size 786432
|
|
- --max-request-tokens 262144
|
|
- --enable-auto-tool-choice
|
|
- --tool-call-parser gemma4
|
|
- --reasoning-parser gemma4
|
|
- --default-chat-template-kwargs '{"enable_thinking":true}'
|
|
residual_issue: Gemma4 thought channel delimiters can still leak into final assistant content on Pi TUI output after successful tool execution.
|
|
containment_preference: prefer a vllm-mlx Gemma-specific Pi extension/provider adapter sanitizer over further runtime option churn if the residual issue appears in assistant content
|
|
queue_timeout_ms: 0
|
|
request_timeout_ms: 1800000
|
|
runtime:
|
|
host: dc-devui-MacStudio.local
|
|
os: macOS 26.2
|
|
hardware: Apple M3 Ultra / 512GB RAM / 80-core GPU
|
|
manager: screen
|
|
manager_status: detached screen session vllm_mlx_8004; DYLD_LIBRARY_PATH=/opt/homebrew/opt/expat/lib is required for pyexpat on this host
|
|
screen_session: vllm_mlx_8004
|
|
start_script: /Users/dc_dev/iop-dev-corp-field/start_vllm_mlx_8004.sh
|
|
python: /Users/dc_dev/vllm-env-0.4.0/bin/python
|
|
package_baseline:
|
|
vllm_mlx: 0.4.0
|
|
mlx: 0.32.0
|
|
mlx_lm: 0.31.3
|
|
mlx_vlm: 0.6.4
|
|
torch: 2.12.1
|
|
transformers: 5.12.1
|
|
huggingface_hub: 1.22.0
|
|
args:
|
|
host: 0.0.0.0
|
|
port: 8004
|
|
continuous_batching: true
|
|
max_num_seqs: 6
|
|
prefill_batch_size: 6
|
|
completion_batch_size: 6
|
|
chunked_prefill_tokens: 1024
|
|
disable_prefix_cache: true
|
|
max_kv_size: 786432
|
|
max_request_tokens: 262144
|
|
context_window_max: 262144
|
|
kv_size_tokens_effective: 786432
|
|
enable_auto_tool_choice: true
|
|
tool_call_parser: gemma4
|
|
reasoning_parser: gemma4
|
|
default_chat_template_kwargs:
|
|
enable_thinking: true
|
|
secondary_provider_candidate:
|
|
id: corp-mac-studio-mlx-vlm-diffusiongemma
|
|
type: mlx-vlm
|
|
endpoint: http://192.168.2.3:8005/v1
|
|
health: http://192.168.2.3:8005/health
|
|
served_model: mlx-community/diffusiongemma-26B-A4B-it-OptiQ-4bit
|
|
include_in_default_pool: false
|
|
note: exposes multiple models and should be routed only after alias/capacity policy is decided
|