From 398c9247dbb1a8c456adb8c2cf0341565ee3c009 Mon Sep 17 00:00:00 2001 From: toki Date: Sun, 12 Jul 2026 15:41:08 +0900 Subject: [PATCH] =?UTF-8?q?docs(dev):=20Ornith=20vLLM=20=EC=84=A4=EC=A0=95?= =?UTF-8?q?=20=EC=9D=B8=EB=B2=A4=ED=86=A0=EB=A6=AC=EB=A5=BC=20=EB=B3=B4?= =?UTF-8?q?=EA=B0=95=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent-test/dev/inventory.yaml | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/agent-test/dev/inventory.yaml b/agent-test/dev/inventory.yaml index 9dc8735..b3a103f 100644 --- a/agent-test/dev/inventory.yaml +++ b/agent-test/dev/inventory.yaml @@ -180,27 +180,103 @@ model: host: toki@192.168.0.91 container_name: iop-vllm-ornith35b-fp8 docker_image: vllm/vllm-openai:nightly-aarch64 + docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f + vllm_image_tag_env: vllm/vllm-openai:nightly-a346d589f5932d4234bf5bf8718f10e26d187021 + vllm_version: 0.23.1rc1.dev223+ga346d589f + vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021 + vllm_build_url: https://buildkite.com/vllm/release-v2/builds/2923 + torch_version: 2.11.0+cu130 + transformers_version: 5.12.1 + compressed_tensors_version: 0.17.0 + flashinfer_python_version: 0.6.12 + cuda_version_env: 13.0.2 + gpu: NVIDIA GB10 + nvidia_driver_version: 580.126.09 endpoint: http://192.168.0.91:8002/v1 hf_model: deepreinforce-ai/Ornith-1.0-35B-FP8 + hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8 + hf_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca + hf_ref: main + revision_argument: not_set_uses_repo_default_main_at_container_start + tokenizer_revision_argument: not_set_uses_model_revision + hf_home: /models/.cache/huggingface + hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca served_model_name: ornith:35b + openai_models_root: deepreinforce-ai/Ornith-1.0-35B-FP8 + openai_models_max_model_len: 262144 quantization: fp8 vllm_quantization_backend: compressed-tensors + model_type: qwen3_5_moe + model_architecture: Qwen3_5MoeForConditionalGeneration + quantization_config_source: model config.json compressed-tensors W8A8 FP8, weight channel strategy, input activation token strategy, linear_attn/embed/lm_head/shared gates ignored dtype: bfloat16 max_model_len: 262144 max_num_seqs: 2 + max_num_batched_tokens: vllm_default_not_overridden gpu_memory_utilization: 0.50 enable_prefix_caching: true + enable_chunked_prefill: true + async_scheduling: true + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + data_parallel_size: 1 + load_format: auto + download_dir: null + kv_cache_dtype: auto + enforce_eager: false + seed: 0 enable_auto_tool_choice: true tool_call_parser: qwen3_xml reasoning_parser: qwen3 + reasoning_parser_enable_in_reasoning: false trust_remote_code: true language_model_only: true + attention_backend: FLASH_ATTN + sampling_backend: FlashInfer top-p/top-k + fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8 + fp8_moe_backend: TRITON + generation_config_override: + temperature: 1.0 + top_k: 20 + top_p: 0.95 + eos_token_id: + - 248046 + - 248044 + bos_token_id: 248044 + docker_runtime: + created_at: "2026-07-12T04:40:26.013681328Z" + entrypoint: + - vllm + - serve + container_port: 8000 + host_port: 8002 + network_mode: bridge + ipc_mode: private + shm_size_bytes: 67108864 + restart_policy: "no" + gpu_device_request: all + bind_mounts: + /home/toki/Data/models: /models + /home/toki/Data/vllm/templates: /templates model_load_memory_gib: 34.43 + model_load_seconds: 611.880026 + weight_download_seconds: 407.868734 + cuda_graph_memory_estimated_gib: 0.13 + cuda_graph_pool_memory_actual_gib: 0.07 available_kv_cache_memory_gib: 21.33 gpu_kv_cache_tokens: 1087640 max_concurrency_for_262144_token_requests: 4.15 + startup_warnings: + - vLLM command does not pin --revision; the recorded hf_revision is the downloaded snapshot observed in cache. + - Prefix caching in Mamba cache align mode is experimental for this architecture. + - Default MoE config for NVIDIA_GB10 FP8 was not found; vLLM used default MoE config, which may affect performance. + - Model generation_config overrides vLLM defaults unless callers provide sampling params or vLLM is relaunched with --generation-config vllm. official_4bit_gguf_status: hf_model: deepreinforce-ai/Ornith-1.0-35B-GGUF + hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-GGUF + hf_revision: c2e1703039380de4ce6820e97afd185682d3c16c + hf_ref: main + hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c file: ornith-1.0-35b-Q4_K_M.gguf status: not_active reason: current vLLM GGUF plugin path exposed /v1/models but failed semantic generation smoke; active working runtime uses the official FP8 vLLM model instead. @@ -434,6 +510,8 @@ nodes: endpoint: http://192.168.0.91:8002/v1 served_model: ornith:35b upstream_model: deepreinforce-ai/Ornith-1.0-35B-FP8 + upstream_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8 + upstream_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca status: active_direct_pi_provider_not_iop_routed pi_model_parameters: context_window: 262144 @@ -442,19 +520,51 @@ nodes: runtime: container_name: iop-vllm-ornith35b-fp8 docker_image: vllm/vllm-openai:nightly-aarch64 + docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f + vllm_version: 0.23.1rc1.dev223+ga346d589f + vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021 port_mapping: 0.0.0.0:8002->8000 + hf_home: /models/.cache/huggingface + hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca quantization: fp8 vllm_quantization_backend: compressed-tensors dtype: bfloat16 max_model_len: 262144 max_num_seqs: 2 + max_num_batched_tokens: vllm_default_not_overridden gpu_memory_utilization: 0.50 enable_prefix_caching: true + enable_chunked_prefill: true + async_scheduling: true + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + data_parallel_size: 1 + load_format: auto + download_dir: null + kv_cache_dtype: auto + enforce_eager: false enable_auto_tool_choice: true tool_call_parser: qwen3_xml reasoning_parser: qwen3 + reasoning_parser_enable_in_reasoning: false trust_remote_code: true language_model_only: true + attention_backend: FLASH_ATTN + fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8 + fp8_moe_backend: TRITON + generation_config_override: + temperature: 1.0 + top_k: 20 + top_p: 0.95 + docker_runtime: + network_mode: bridge + ipc_mode: private + shm_size_bytes: 67108864 + restart_policy: "no" + gpu_device_request: all + bind_mounts: + /home/toki/Data/models: /models + /home/toki/Data/vllm/templates: /templates model_load_memory_gib: 34.43 available_kv_cache_memory_gib: 21.33 gpu_kv_cache_tokens: 1087640