Initial import: bare-metal stacks, Server UI, GPU fan, tutorials.

Infrastructure configs for GMKtec K11 (Docker, vLLM, LocalAI, ComfyUI, control-plane, gpu-fan agent, Server UI with CLI/file explorer/GPU fan curve). Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-05 12:02:04 +00:00
commit 359afb3a59
153 changed files with 18169 additions and 0 deletions
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Merge KV cache settings (q8_0) into a model YAML on /data.
+# Usage: ./scripts/apply-kv-profile.sh <model-name-without-.yaml>
+# Example: ./scripts/apply-kv-profile.sh gemma-4-12b-it-qat-q4_0
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+if [[ ! -f "${STACK_DIR}/.env" ]]; then
+  echo "ERROR: ${STACK_DIR}/.env not found"
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1091
+source "${STACK_DIR}/.env"
+set +a
+
+DATA_ROOT="${DATA_ROOT:-/data}"
+MODELS_DIR="${DATA_ROOT}/apps/localai/models"
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <model-name>"
+  echo "Example: $0 gemma-4-12b-it-qat-q4_0"
+  exit 1
+fi
+
+MODEL_NAME="$1"
+TARGET="${MODELS_DIR}/${MODEL_NAME}.yaml"
+
+if [[ ! -f "${TARGET}" ]]; then
+  echo "ERROR: ${TARGET} not found"
+  exit 1
+fi
+
+BACKUP="${TARGET}.bak.$(date +%Y%m%d%H%M%S)"
+cp "${TARGET}" "${BACKUP}"
+echo "Backup: ${BACKUP}"
+
+run_python_merge() {
+  python3 - "${1}" <<'PY'
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    sys.exit("ERROR: python3-yaml required (sudo apt install python3-yaml)")
+
+path = Path(sys.argv[1])
+data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+
+params = data.setdefault("parameters", {})
+kv = {
+    "cache_type_k": "q8_0",
+    "cache_type_v": "q8_0",
+    "flash_attention": True,
+}
+if "context_size" not in params:
+    kv["context_size"] = 8192
+params.update(kv)
+
+path.write_text(
+    yaml.dump(data, default_flow_style=False, allow_unicode=True, sort_keys=False),
+    encoding="utf-8",
+)
+print(f"Updated KV settings in {path}")
+for k, v in kv.items():
+    print(f"  {k}: {v}")
+if "context_size" in params and "context_size" not in kv:
+    print(f"  context_size: {params['context_size']} (unchanged)")
+PY
+}
+
+if run_python_merge "${TARGET}" 2>/dev/null; then
+  :
+elif docker ps --format '{{.Names}}' 2>/dev/null | grep -qx localai; then
+  echo "Host write failed — applying via docker exec localai (models volume is root-owned)"
+  CONTAINER_PATH="/models/${MODEL_NAME}.yaml"
+  docker exec localai sh -c "grep -q cache_type_k '${CONTAINER_PATH}' || sed -i '/^    model:/a\\
+    cache_type_k: q8_0\\
+    cache_type_v: q8_0\\
+    flash_attention: true\\
+    context_size: 8192' '${CONTAINER_PATH}'"
+  grep -E 'cache_type_k|cache_type_v|flash_attention|context_size' "${TARGET}" || true
+else
+  echo "ERROR: cannot write ${TARGET} (permission denied) and container localai not running"
+  echo "Run: sudo $0 ${MODEL_NAME}"
+  exit 1
+fi
+
+echo ""
+echo "Restart LocalAI to apply:"
+echo "  cd ${STACK_DIR} && docker compose --profile localai restart localai"
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+UPSTREAM_DIR="${STACK_DIR}/upstream"
+REPO_URL="https://github.com/mudler/LocalAI.git"
+TAG="v4.4.3"
+
+cd "${STACK_DIR}"
+
+if [[ -d "${UPSTREAM_DIR}/.git" ]]; then
+  echo "Upstream already cloned: ${UPSTREAM_DIR}"
+  echo "Remove it first to re-clone: rm -rf upstream"
+  exit 0
+fi
+
+echo "=== Cloning LocalAI upstream (reference only) ==="
+echo "Repo: ${REPO_URL}"
+echo "Tag:  ${TAG}"
+echo "Dest: ${UPSTREAM_DIR}"
+echo ""
+
+git clone --depth 1 --branch "${TAG}" "${REPO_URL}" "${UPSTREAM_DIR}"
+
+echo ""
+echo "Done. Use upstream/ for example model YAML and compose reference."
+echo "Runtime uses the official Docker image from .env — not a local build."
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Download BGE-Reranker-v2-m3 FP16 GGUF and apply YAML profile.
+# Usage: ./scripts/download-reranker.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+cd "${STACK_DIR}"
+
+if [[ -f .env ]]; then
+  set -a
+  # shellcheck disable=SC1091
+  source .env
+  set +a
+fi
+
+DATA_ROOT="${DATA_ROOT:-/data}"
+MODELS_DIR="${DATA_ROOT}/apps/localai/models"
+GGUF_NAME="bge-reranker-v2-m3-FP16.gguf"
+GGUF_URL="https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF/resolve/main/${GGUF_NAME}"
+YAML_SRC="${STACK_DIR}/profiles/bge-reranker-v2-m3-FP16-rerank.yaml.example"
+YAML_DST="${MODELS_DIR}/${GGUF_NAME}.yaml"
+
+"${SCRIPT_DIR}/ensure-dirs.sh" "${DATA_ROOT}"
+
+echo "=== BGE-Reranker-v2-m3 download ==="
+echo "Target: ${MODELS_DIR}/${GGUF_NAME}"
+echo ""
+
+if [[ -f "${MODELS_DIR}/${GGUF_NAME}" ]]; then
+  echo "GGUF already exists — skipping download"
+else
+  if command -v wget &>/dev/null; then
+    wget -c -O "${MODELS_DIR}/${GGUF_NAME}.partial" "${GGUF_URL}"
+    mv "${MODELS_DIR}/${GGUF_NAME}.partial" "${MODELS_DIR}/${GGUF_NAME}"
+  else
+    curl -fL -C - -o "${MODELS_DIR}/${GGUF_NAME}.partial" "${GGUF_URL}"
+    mv "${MODELS_DIR}/${GGUF_NAME}.partial" "${MODELS_DIR}/${GGUF_NAME}"
+  fi
+fi
+
+echo ""
+echo "=== Applying YAML profile ==="
+if cp "${YAML_SRC}" "${YAML_DST}" 2>/dev/null; then
+  chmod 644 "${YAML_DST}" "${MODELS_DIR}/${GGUF_NAME}" 2>/dev/null || true
+elif docker ps --format '{{.Names}}' | grep -qx localai; then
+  echo "Host copy failed (permissions) — writing via docker exec localai"
+  docker exec -i localai sh -c "cat > /models/${GGUF_NAME}.yaml" < "${YAML_SRC}"
+else
+  echo "ERROR: cannot write ${YAML_DST} (permission denied) and localai container not running"
+  exit 1
+fi
+
+ls -lh "${MODELS_DIR}/${GGUF_NAME}" "${YAML_DST}"
+
+echo ""
+echo "=== Done ==="
+echo "Restart LocalAI to load the model:"
+echo "  cd ${STACK_DIR} && docker compose --profile localai restart localai"
+echo ""
+echo "Smoke test:"
+echo '  curl -s http://127.0.0.1:${LOCALAI_PORT:-8070}/v1/rerank \'
+echo '    -H "Authorization: Bearer <LOCALAI_API_KEY>" \'
+echo '    -H "Content-Type: application/json" \'
+echo '    -d '"'"'{"model":"bge-reranker-v2-m3-FP16.gguf","query":"panda","documents":["hi","it is a bear","The giant panda is a bear species endemic to China."],"top_n":2}'"'"
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Create LocalAI data directories on the data disk.
+
+ensure_localai_dirs() {
+  local data_root="${1:-/data}"
+  mkdir -p \
+    "${data_root}/apps/localai/models" \
+    "${data_root}/apps/localai/backends" \
+    "${data_root}/apps/localai/configuration" \
+    "${data_root}/apps/localai/images" \
+    "${data_root}/apps/localai/data"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  ensure_localai_dirs "${1:-/data}"
+fi
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+cd "${STACK_DIR}"
+
+if [[ ! -f .env ]]; then
+  echo "ERROR: .env not found. Run: cp .env.example .env"
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1091
+source .env
+set +a
+
+if ! docker info &>/dev/null; then
+  echo "ERROR: Docker is not running"
+  exit 1
+fi
+
+echo "=== LocalAI — pull image only (no start) ==="
+echo "Image: ${LOCALAI_IMAGE:-localai/localai:v4.4.3-gpu-nvidia-cuda-13}"
+echo ""
+
+docker compose --profile localai pull
+
+echo ""
+echo "Done. Start with: ./scripts/start.sh"
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/ensure-dirs.sh"
+
+cd "${STACK_DIR}"
+
+if [[ ! -f .env ]]; then
+  echo "ERROR: .env not found. Run: cp .env.example .env"
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1091
+source .env
+set +a
+
+DATA_ROOT="${DATA_ROOT:-/data}"
+
+if ! mountpoint -q "${DATA_ROOT}" 2>/dev/null; then
+  echo "ERROR: ${DATA_ROOT} is not mounted"
+  echo "       Complete disk setup (tutorial 04 part A) first"
+  exit 1
+fi
+
+ensure_localai_dirs "${DATA_ROOT}"
+
+if ! docker info &>/dev/null; then
+  echo "ERROR: Docker is not running"
+  exit 1
+fi
+
+echo "=== LocalAI stack ==="
+echo "Image:     ${LOCALAI_IMAGE:-localai/localai:v4.4.3-gpu-nvidia-cuda-13}"
+echo "Port:      ${LOCALAI_PORT:-8080}"
+echo "Models:    ${DATA_ROOT}/apps/localai/models (empty OK)"
+echo "GPU:       CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}"
+echo ""
+
+docker compose --profile localai pull
+docker compose --profile localai up -d
+
+echo ""
+echo "Started. Follow logs:"
+echo "  docker compose --profile localai logs -f localai"
+echo ""
+echo "Health check:"
+echo "  curl -s http://localhost:${LOCALAI_PORT:-8080}/readyz"
+echo ""
+echo "Web UI:"
+echo "  http://localhost:${LOCALAI_PORT:-8080}"