Initial import: bare-metal stacks, Server UI, GPU fan, tutorials.
Infrastructure configs for GMKtec K11 (Docker, vLLM, LocalAI, ComfyUI, control-plane, gpu-fan agent, Server UI with CLI/file explorer/GPU fan curve). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
tomasz-syn-grzegorza
@@ -0,0 +1,358 @@
|
||||
"""NVML fan control with JSON curve, modes, and graceful shutdown."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pynvml
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MIN_CURVE_POINTS = 3
|
||||
MAX_CURVE_POINTS = 7
|
||||
MIN_FAN_SPEED = 30
|
||||
MAX_FAN_SPEED = 100
|
||||
|
||||
|
||||
class FanControlError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _clamp_speed(speed: int) -> int:
|
||||
if speed == 0:
|
||||
return 0
|
||||
return max(MIN_FAN_SPEED, min(MAX_FAN_SPEED, speed))
|
||||
|
||||
|
||||
def parse_curve(raw: dict[str, Any]) -> list[tuple[int, int]]:
|
||||
if not raw:
|
||||
raise FanControlError("Curve is empty")
|
||||
|
||||
points: list[tuple[int, int]] = []
|
||||
for temp_str, speed in raw.items():
|
||||
try:
|
||||
temp = int(temp_str)
|
||||
speed_int = int(speed)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise FanControlError(f"Invalid curve point {temp_str!r}: {speed!r}") from exc
|
||||
|
||||
if not 0 <= temp <= 120:
|
||||
raise FanControlError(f"Temperature {temp} out of range 0-120")
|
||||
if speed_int != 0 and not MIN_FAN_SPEED <= speed_int <= MAX_FAN_SPEED:
|
||||
raise FanControlError(
|
||||
f"Fan speed {speed_int}% invalid (use 0 or {MIN_FAN_SPEED}-{MAX_FAN_SPEED})"
|
||||
)
|
||||
points.append((temp, speed_int))
|
||||
|
||||
points.sort(key=lambda p: p[0])
|
||||
temps = [p[0] for p in points]
|
||||
if len(set(temps)) != len(temps):
|
||||
raise FanControlError("Temperatures must be unique")
|
||||
if len(points) < MIN_CURVE_POINTS or len(points) > MAX_CURVE_POINTS:
|
||||
raise FanControlError(
|
||||
f"Curve must have {MIN_CURVE_POINTS}-{MAX_CURVE_POINTS} points"
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
def curve_to_dict(points: list[tuple[int, int]]) -> dict[str, int]:
|
||||
return {str(temp): speed for temp, speed in points}
|
||||
|
||||
|
||||
def interpolate_speed(temp: int, curve: list[tuple[int, int]]) -> int:
|
||||
if temp <= curve[0][0]:
|
||||
return curve[0][1]
|
||||
if temp >= curve[-1][0]:
|
||||
return curve[-1][1]
|
||||
|
||||
for i in range(len(curve) - 1):
|
||||
t1, s1 = curve[i]
|
||||
t2, s2 = curve[i + 1]
|
||||
if t1 <= temp <= t2:
|
||||
if t2 == t1:
|
||||
return s2
|
||||
ratio = (temp - t1) / (t2 - t1)
|
||||
return int(s1 + ratio * (s2 - s1))
|
||||
return curve[-1][1]
|
||||
|
||||
|
||||
class FanController:
|
||||
def __init__(
|
||||
self,
|
||||
curve_path: Path,
|
||||
gpu_index: int = 0,
|
||||
poll_interval: float = 2.0,
|
||||
) -> None:
|
||||
self.curve_path = curve_path
|
||||
self.gpu_index = gpu_index
|
||||
self.poll_interval = poll_interval
|
||||
self._lock = threading.Lock()
|
||||
self._running = False
|
||||
self._mode = "curve"
|
||||
self._manual_speed = 100
|
||||
self._curve: list[tuple[int, int]] = []
|
||||
self._handle: Any = None
|
||||
self._fan_count = 0
|
||||
self._gpu_name = ""
|
||||
self._last_metrics: dict[str, Any] = {}
|
||||
self._auto_active = False
|
||||
self.dry_run = False
|
||||
|
||||
@property
|
||||
def mode(self) -> str:
|
||||
with self._lock:
|
||||
return self._mode
|
||||
|
||||
@property
|
||||
def manual_speed(self) -> int:
|
||||
with self._lock:
|
||||
return self._manual_speed
|
||||
|
||||
def load_curve_file(self) -> list[tuple[int, int]]:
|
||||
if not self.curve_path.exists():
|
||||
raise FanControlError(f"Curve file not found: {self.curve_path}")
|
||||
raw = json.loads(self.curve_path.read_text(encoding="utf-8"))
|
||||
return parse_curve(raw)
|
||||
|
||||
def save_curve_file(self, points: list[tuple[int, int]]) -> None:
|
||||
validated = parse_curve(curve_to_dict(points))
|
||||
self.curve_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.curve_path.write_text(
|
||||
json.dumps(curve_to_dict(validated), indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self._lock:
|
||||
self._curve = validated
|
||||
log.info("Saved curve: %s", validated)
|
||||
|
||||
def reload_curve(self) -> None:
|
||||
curve = self.load_curve_file()
|
||||
with self._lock:
|
||||
self._curve = curve
|
||||
log.info("Reloaded curve: %s", curve)
|
||||
|
||||
def set_mode(self, mode: str, manual_speed: int | None = None) -> None:
|
||||
if mode not in ("auto", "curve", "manual"):
|
||||
raise FanControlError(f"Unknown mode: {mode}")
|
||||
with self._lock:
|
||||
self._mode = mode
|
||||
if manual_speed is not None:
|
||||
self._manual_speed = _clamp_speed(manual_speed)
|
||||
log.info("Mode set to %s (manual_speed=%s)", mode, manual_speed)
|
||||
|
||||
def get_curve(self) -> list[tuple[int, int]]:
|
||||
with self._lock:
|
||||
return list(self._curve)
|
||||
|
||||
def get_metrics(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
return dict(self._last_metrics)
|
||||
|
||||
def init_nvml(self) -> None:
|
||||
pynvml.nvmlInit()
|
||||
count = pynvml.nvmlDeviceGetCount()
|
||||
if self.gpu_index >= count:
|
||||
raise FanControlError(
|
||||
f"GPU index {self.gpu_index} out of range (found {count} GPU(s))"
|
||||
)
|
||||
|
||||
handle = pynvml.nvmlDeviceGetHandleByIndex(self.gpu_index)
|
||||
name = pynvml.nvmlDeviceGetName(handle)
|
||||
fan_count = pynvml.nvmlDeviceGetNumFans(handle)
|
||||
|
||||
self._handle = handle
|
||||
self._fan_count = fan_count
|
||||
self._gpu_name = name if isinstance(name, str) else name.decode()
|
||||
self.reload_curve()
|
||||
log.info("GPU %d: %s (%d fan(s))", self.gpu_index, self._gpu_name, fan_count)
|
||||
|
||||
def _read_metrics(self) -> dict[str, Any]:
|
||||
handle = self._handle
|
||||
assert handle is not None
|
||||
|
||||
temp = pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU)
|
||||
util = pynvml.nvmlDeviceGetUtilizationRates(handle)
|
||||
power_mw = pynvml.nvmlDeviceGetPowerUsage(handle)
|
||||
|
||||
mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
|
||||
memory_used_mb = mem_info.used // (1024**2)
|
||||
memory_total_mb = mem_info.total // (1024**2)
|
||||
|
||||
power_limit_w: float | None = None
|
||||
clock_graphics_mhz: int | None = None
|
||||
clock_memory_mhz: int | None = None
|
||||
try:
|
||||
power_limit_w = round(
|
||||
pynvml.nvmlDeviceGetEnforcedPowerLimit(handle) / 1000, 1
|
||||
)
|
||||
except pynvml.NVMLError:
|
||||
pass
|
||||
try:
|
||||
clock_graphics_mhz = pynvml.nvmlDeviceGetClockInfo(
|
||||
handle, pynvml.NVML_CLOCK_GRAPHICS
|
||||
)
|
||||
except pynvml.NVMLError:
|
||||
pass
|
||||
try:
|
||||
clock_memory_mhz = pynvml.nvmlDeviceGetClockInfo(
|
||||
handle, pynvml.NVML_CLOCK_MEM
|
||||
)
|
||||
except pynvml.NVMLError:
|
||||
pass
|
||||
|
||||
fan_speeds: list[int] = []
|
||||
for fan_idx in range(self._fan_count):
|
||||
try:
|
||||
fan_speeds.append(pynvml.nvmlDeviceGetFanSpeed_v2(handle, fan_idx))
|
||||
except pynvml.NVMLError:
|
||||
fan_speeds.append(-1)
|
||||
|
||||
with self._lock:
|
||||
mode = self._mode
|
||||
manual_speed = self._manual_speed
|
||||
curve = list(self._curve)
|
||||
|
||||
if mode == "curve":
|
||||
target = interpolate_speed(temp, curve)
|
||||
elif mode == "manual":
|
||||
target = manual_speed
|
||||
else:
|
||||
target = None
|
||||
|
||||
metrics = {
|
||||
"gpu_index": self.gpu_index,
|
||||
"gpu_name": self._gpu_name,
|
||||
"temperature_c": temp,
|
||||
"fan_speeds_pct": fan_speeds,
|
||||
"target_speed_pct": target,
|
||||
"power_w": round(power_mw / 1000, 1),
|
||||
"power_limit_w": power_limit_w,
|
||||
"utilization_pct": util.gpu,
|
||||
"memory_utilization_pct": util.memory,
|
||||
"memory_used_mb": memory_used_mb,
|
||||
"memory_total_mb": memory_total_mb,
|
||||
"clock_graphics_mhz": clock_graphics_mhz,
|
||||
"clock_memory_mhz": clock_memory_mhz,
|
||||
"mode": mode,
|
||||
"manual_speed_pct": manual_speed,
|
||||
"curve": curve_to_dict(curve),
|
||||
}
|
||||
with self._lock:
|
||||
self._last_metrics = metrics
|
||||
return metrics
|
||||
|
||||
def _set_manual_policy(self) -> None:
|
||||
handle = self._handle
|
||||
assert handle is not None
|
||||
for fan_idx in range(self._fan_count):
|
||||
pynvml.nvmlDeviceSetFanControlPolicy(
|
||||
handle, fan_idx, pynvml.NVML_FAN_POLICY_MANUAL
|
||||
)
|
||||
|
||||
def _restore_auto_policy(self) -> None:
|
||||
if self._handle is None:
|
||||
return
|
||||
log.info("Restoring automatic fan control...")
|
||||
for fan_idx in range(self._fan_count):
|
||||
try:
|
||||
pynvml.nvmlDeviceSetFanControlPolicy(
|
||||
self._handle,
|
||||
fan_idx,
|
||||
pynvml.NVML_FAN_POLICY_TEMPERATURE_CONTINOUS_SW,
|
||||
)
|
||||
log.info("Fan %d: restored to auto", fan_idx)
|
||||
except pynvml.NVMLError as exc:
|
||||
log.error("Fan %d: could not restore auto: %s", fan_idx, exc)
|
||||
|
||||
def _apply_fan_speed(self, speed: int) -> None:
|
||||
handle = self._handle
|
||||
assert handle is not None
|
||||
speed = _clamp_speed(speed)
|
||||
self._set_manual_policy()
|
||||
for fan_idx in range(self._fan_count):
|
||||
pynvml.nvmlDeviceSetFanSpeed_v2(handle, fan_idx, speed)
|
||||
|
||||
def update_once(self) -> dict[str, Any]:
|
||||
metrics = self._read_metrics()
|
||||
mode = metrics["mode"]
|
||||
|
||||
if mode == "auto":
|
||||
if not self._auto_active and not self.dry_run:
|
||||
self._restore_auto_policy()
|
||||
self._auto_active = True
|
||||
elif self.dry_run:
|
||||
self._auto_active = True
|
||||
return metrics
|
||||
|
||||
self._auto_active = False
|
||||
target = metrics["target_speed_pct"]
|
||||
if target is None:
|
||||
return metrics
|
||||
|
||||
if self.dry_run:
|
||||
log.info(
|
||||
"[dry-run] GPU %d: %d°C -> %d%% (fans: %s)",
|
||||
self.gpu_index,
|
||||
metrics["temperature_c"],
|
||||
target,
|
||||
metrics["fan_speeds_pct"],
|
||||
)
|
||||
return metrics
|
||||
|
||||
try:
|
||||
self._apply_fan_speed(target)
|
||||
log.info(
|
||||
"GPU %d: %d°C -> %d%% (fans: %s)",
|
||||
self.gpu_index,
|
||||
metrics["temperature_c"],
|
||||
target,
|
||||
metrics["fan_speeds_pct"],
|
||||
)
|
||||
except pynvml.NVMLError as exc:
|
||||
log.error("Failed to set fan speed: %s", exc)
|
||||
metrics["error"] = str(exc)
|
||||
return metrics
|
||||
|
||||
def run_loop(self) -> None:
|
||||
self._running = True
|
||||
log.info("Fan control loop started (interval=%ss)", self.poll_interval)
|
||||
while self._running:
|
||||
try:
|
||||
self.update_once()
|
||||
except Exception:
|
||||
log.exception("Fan control loop error")
|
||||
time.sleep(self.poll_interval)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stop()
|
||||
if not self.dry_run:
|
||||
self._restore_auto_policy()
|
||||
try:
|
||||
pynvml.nvmlShutdown()
|
||||
except pynvml.NVMLError:
|
||||
pass
|
||||
log.info("Fan controller shut down")
|
||||
|
||||
def install_signal_handlers(self) -> None:
|
||||
def handler(signum: int, _frame: Any) -> None:
|
||||
log.info("Received signal %s", signum)
|
||||
if signum == signal.SIGHUP:
|
||||
try:
|
||||
self.reload_curve()
|
||||
except FanControlError as exc:
|
||||
log.error("Curve reload failed: %s", exc)
|
||||
return
|
||||
self.shutdown()
|
||||
|
||||
signal.signal(signal.SIGTERM, handler)
|
||||
signal.signal(signal.SIGINT, handler)
|
||||
signal.signal(signal.SIGHUP, handler)
|
||||
Reference in New Issue
Block a user