toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

docs(dev): Ornith vLLM 설정 인벤토리를 보강한다

Browse source
This commit is contained in:
toki 2026-07-12 15:41:08 +09:00
parent 9578744c35
commit 398c9247db
1 changed files with 110 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
agent-test/dev/inventory.yaml
View file

@ -180,27 +180,103 @@ model:
host: toki@192.168.0.91
container_name: iop-vllm-ornith35b-fp8
docker_image: vllm/vllm-openai:nightly-aarch64
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
vllm_image_tag_env: vllm/vllm-openai:nightly-a346d589f5932d4234bf5bf8718f10e26d187021
vllm_version: 0.23.1rc1.dev223+ga346d589f
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
vllm_build_url: https://buildkite.com/vllm/release-v2/builds/2923
torch_version: 2.11.0+cu130
transformers_version: 5.12.1
compressed_tensors_version: 0.17.0
flashinfer_python_version: 0.6.12
cuda_version_env: 13.0.2
gpu: NVIDIA GB10
nvidia_driver_version: 580.126.09
endpoint: http://192.168.0.91:8002/v1
hf_model: deepreinforce-ai/Ornith-1.0-35B-FP8
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
hf_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
hf_ref: main
revision_argument: not_set_uses_repo_default_main_at_container_start
tokenizer_revision_argument: not_set_uses_model_revision
hf_home: /models/.cache/huggingface
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
served_model_name: ornith:35b
openai_models_root: deepreinforce-ai/Ornith-1.0-35B-FP8
openai_models_max_model_len: 262144
quantization: fp8
vllm_quantization_backend: compressed-tensors
model_type: qwen3_5_moe
model_architecture: Qwen3_5MoeForConditionalGeneration
quantization_config_source: model config.json compressed-tensors W8A8 FP8, weight channel strategy, input activation token strategy, linear_attn/embed/lm_head/shared gates ignored
dtype: bfloat16
max_model_len: 262144
max_num_seqs: 2
max_num_batched_tokens: vllm_default_not_overridden
gpu_memory_utilization: 0.50
enable_prefix_caching: true
enable_chunked_prefill: true
async_scheduling: true
tensor_parallel_size: 1
pipeline_parallel_size: 1
data_parallel_size: 1
load_format: auto
download_dir: null
kv_cache_dtype: auto
enforce_eager: false
seed: 0
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
reasoning_parser_enable_in_reasoning: false
trust_remote_code: true
language_model_only: true
attention_backend: FLASH_ATTN
sampling_backend: FlashInfer top-p/top-k
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
fp8_moe_backend: TRITON
generation_config_override:
temperature: 1.0
top_k: 20
top_p: 0.95
eos_token_id:
- 248046
- 248044
bos_token_id: 248044
docker_runtime:
created_at: "2026-07-12T04:40:26.013681328Z"
entrypoint:
- vllm
- serve
container_port: 8000
host_port: 8002
network_mode: bridge
ipc_mode: private
shm_size_bytes: 67108864
restart_policy: "no"
gpu_device_request: all
bind_mounts:
/home/toki/Data/models: /models
/home/toki/Data/vllm/templates: /templates
model_load_memory_gib: 34.43
model_load_seconds: 611.880026
weight_download_seconds: 407.868734
cuda_graph_memory_estimated_gib: 0.13
cuda_graph_pool_memory_actual_gib: 0.07
available_kv_cache_memory_gib: 21.33
gpu_kv_cache_tokens: 1087640
max_concurrency_for_262144_token_requests: 4.15
startup_warnings:
- vLLM command does not pin --revision; the recorded hf_revision is the downloaded snapshot observed in cache.
- Prefix caching in Mamba cache align mode is experimental for this architecture.
- Default MoE config for NVIDIA_GB10 FP8 was not found; vLLM used default MoE config, which may affect performance.
- Model generation_config overrides vLLM defaults unless callers provide sampling params or vLLM is relaunched with --generation-config vllm.
official_4bit_gguf_status:
hf_model: deepreinforce-ai/Ornith-1.0-35B-GGUF
hf_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-GGUF
hf_revision: c2e1703039380de4ce6820e97afd185682d3c16c
hf_ref: main
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c
file: ornith-1.0-35b-Q4_K_M.gguf
status: not_active
reason: current vLLM GGUF plugin path exposed /v1/models but failed semantic generation smoke; active working runtime uses the official FP8 vLLM model instead.
@ -434,6 +510,8 @@ nodes:
endpoint: http://192.168.0.91:8002/v1
served_model: ornith:35b
upstream_model: deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_url: https://huggingface.co/deepreinforce-ai/Ornith-1.0-35B-FP8
upstream_revision: 1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
status: active_direct_pi_provider_not_iop_routed
pi_model_parameters:
context_window: 262144
@ -442,19 +520,51 @@ nodes:
runtime:
container_name: iop-vllm-ornith35b-fp8
docker_image: vllm/vllm-openai:nightly-aarch64
docker_image_id: sha256:a720df3e84a89d7db47a3b7a0511cb5b312e203fc4956f7493df248299267a6f
vllm_version: 0.23.1rc1.dev223+ga346d589f
vllm_build_commit: a346d589f5932d4234bf5bf8718f10e26d187021
port_mapping: 0.0.0.0:8002->8000
hf_home: /models/.cache/huggingface
hf_cache_snapshot_path: /models/.cache/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-FP8/snapshots/1ab57ce0b44950e498a88756f40ad1ed4d0f30ca
quantization: fp8
vllm_quantization_backend: compressed-tensors
dtype: bfloat16
max_model_len: 262144
max_num_seqs: 2
max_num_batched_tokens: vllm_default_not_overridden
gpu_memory_utilization: 0.50
enable_prefix_caching: true
enable_chunked_prefill: true
async_scheduling: true
tensor_parallel_size: 1
pipeline_parallel_size: 1
data_parallel_size: 1
load_format: auto
download_dir: null
kv_cache_dtype: auto
enforce_eager: false
enable_auto_tool_choice: true
tool_call_parser: qwen3_xml
reasoning_parser: qwen3
reasoning_parser_enable_in_reasoning: false
trust_remote_code: true
language_model_only: true
attention_backend: FLASH_ATTN
fp8_linear_kernel: CutlassFP8ScaledMMLinearKernel for CompressedTensorsW8A8Fp8
fp8_moe_backend: TRITON
generation_config_override:
temperature: 1.0
top_k: 20
top_p: 0.95
docker_runtime:
network_mode: bridge
ipc_mode: private
shm_size_bytes: 67108864
restart_policy: "no"
gpu_device_request: all
bind_mounts:
/home/toki/Data/models: /models
/home/toki/Data/vllm/templates: /templates
model_load_memory_gib: 34.43
available_kv_cache_memory_gib: 21.33
gpu_kv_cache_tokens: 1087640