tomasz-syn-grzegorza
359afb3a59
Infrastructure configs for GMKtec K11 (Docker, vLLM, LocalAI, ComfyUI, control-plane, gpu-fan agent, Server UI with CLI/file explorer/GPU fan curve). Co-authored-by: Cursor <cursoragent@cursor.com>
27 lines
618 B
Bash
Executable File
27 lines
618 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ARGS=(
|
|
--model "${VLLM_MODEL}"
|
|
--served-model-name "${SERVED_MODEL_NAME:-qwen3.6-27b}"
|
|
--host 0.0.0.0
|
|
--port 8000
|
|
--max-model-len "${MAX_MODEL_LEN:-131072}"
|
|
--max-num-seqs "${MAX_NUM_SEQS:-1}"
|
|
--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.95}"
|
|
--kv-cache-dtype "${KV_CACHE_DTYPE:-fp8}"
|
|
--trust-remote-code
|
|
)
|
|
|
|
if [[ -n "${QUANTIZATION:-}" ]]; then
|
|
ARGS+=(--quantization "${QUANTIZATION}")
|
|
fi
|
|
|
|
if [[ -n "${VLLM_EXTRA_ARGS:-}" ]]; then
|
|
# shellcheck disable=SC2206
|
|
EXTRA=( ${VLLM_EXTRA_ARGS} )
|
|
ARGS+=("${EXTRA[@]}")
|
|
fi
|
|
|
|
exec vllm serve "${ARGS[@]}"
|