Chapter 13 - The Local Model Stack

This chapter inventories the local hardware, llama.cpp runtime, and GGUF files before any modeling work. The point is to make fit decisions from observed files and machine limits rather than from model names alone.

from __future__ import annotations

import os
import re
import shutil
import subprocess
from pathlib import Path

import pandas as pd

pd.set_option("display.max_colwidth", 120)

LLAMA_ROOT = Path.home() / "tmp" / "llama-cpp"
LLAMA_CPP = LLAMA_ROOT / "llama.cpp"
BIN_DIR = LLAMA_CPP / "build" / "bin"
MODEL_DIR = LLAMA_ROOT / "models"
LLAMA_SERVER = BIN_DIR / "llama-server"
LLAMA_CLI = BIN_DIR / "llama-cli"
LLAMA_BENCH = BIN_DIR / "llama-bench"

for name, path in {
    "llama.cpp root": LLAMA_CPP,
    "binary directory": BIN_DIR,
    "model directory": MODEL_DIR,
    "llama-server": LLAMA_SERVER,
}.items():
    print(f"{name}: {path} ({'found' if path.exists() else 'missing'})")
llama.cpp root: /home/alal/tmp/llama-cpp/llama.cpp (found)
binary directory: /home/alal/tmp/llama-cpp/llama.cpp/build/bin (found)
model directory: /home/alal/tmp/llama-cpp/models (found)
llama-server: /home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-server (found)
def run_command(command: list[str], timeout: int = 15) -> str:
    result = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
    text = (result.stdout or result.stderr).strip()
    if result.returncode != 0:
        return f"command failed with code {result.returncode}: {text}"
    return text

print("GPU")
print(run_command([
    "nvidia-smi",
    "--query-gpu=name,memory.total,driver_version",
    "--format=csv,noheader,nounits",
]))

print("\nSystem memory")
print(run_command(["free", "-h"]))

print("\nllama-server build")
print(run_command([str(LLAMA_SERVER), "--version"]))
GPU
NVIDIA GeForce RTX 5070, 12227, 580.173.02

System memory
total        used        free      shared  buff/cache   available
Mem:            31Gi        14Gi       355Mi       2.4Gi        19Gi        16Gi
Swap:          511Mi       511Mi       172Ki

llama-server build
version: 1458 (720d7fa40)
built with GNU 13.3.0 for Linux x86_64
def quant_label(path: Path) -> str:
    name = path.name.upper()
    patterns = ["Q2_K", "Q3_K", "Q4_K_M", "Q4_K_S", "Q5_K_M", "Q5_K_S", "Q6_K", "Q8_0", "BF16", "F16", "MXFP4"]
    for pattern in patterns:
        if pattern in name:
            return pattern
    if path.stat().st_size < 100_000_000:
        return "adapter/control-vector"
    return "unknown"

def size_gib(path: Path) -> float:
    return path.stat().st_size / 1024**3

models = sorted(MODEL_DIR.glob("*.gguf"), key=lambda path: path.stat().st_size, reverse=True)
model_inventory = pd.DataFrame([
    {
        "file": path.name,
        "quant_or_format": quant_label(path),
        "size_gib": round(size_gib(path), 2),
        "path": str(path),
    }
    for path in models
])
model_inventory
file quant_or_format size_gib path
0 gpt-oss-20b-mxfp4.gguf MXFP4 11.28 /home/alal/tmp/llama-cpp/models/gpt-oss-20b-mxfp4.gguf
1 talkie-1930-13b-it-Q4_K_M.gguf Q4_K_M 7.98 /home/alal/tmp/llama-cpp/models/talkie-1930-13b-it-Q4_K_M.gguf
2 talkie-Q4_K_M.gguf Q4_K_M 7.87 /home/alal/tmp/llama-cpp/models/talkie-Q4_K_M.gguf
3 Qwen_Qwen3.5-4B-BF16.gguf BF16 7.85 /home/alal/tmp/llama-cpp/models/Qwen_Qwen3.5-4B-BF16.gguf
4 gemma-4-e4b-it-Q8_0.gguf Q8_0 7.48 /home/alal/tmp/llama-cpp/models/gemma-4-e4b-it-Q8_0.gguf
5 Qwen_Qwen3.5-4B-Q4_K_M.gguf Q4_K_M 2.81 /home/alal/tmp/llama-cpp/models/Qwen_Qwen3.5-4B-Q4_K_M.gguf
6 qwen35_4b_steam_v1_playtime_lora-f16.gguf F16 0.01 /home/alal/tmp/llama-cpp/models/qwen35_4b_steam_v1_playtime_lora-f16.gguf
7 qwen35_4b_rich_history_global_layer24_cvector.gguf adapter/control-vector 0.00 /home/alal/tmp/llama-cpp/models/qwen35_4b_rich_history_global_layer24_cvector.gguf 
def system_memory_gib() -> float:
    meminfo = Path("/proc/meminfo").read_text().splitlines()
    total_kib = next(int(line.split()[1]) for line in meminfo if line.startswith("MemTotal:"))
    return total_kib / 1024**2

def gpu_memory_gib() -> float | None:
    output = run_command([
        "nvidia-smi",
        "--query-gpu=memory.total",
        "--format=csv,noheader,nounits",
    ])
    try:
        return float(output.splitlines()[0].strip()) / 1024
    except Exception:
        return None

system_ram_gib = system_memory_gib()
gpu_vram_gib = gpu_memory_gib()
usable_vram_gib = (gpu_vram_gib or 0.0) - 1.5

def fit_assessment(size: float, label: str) -> str:
    if label == "adapter/control-vector":
        return "sidecar artifact, not a standalone model"
    if gpu_vram_gib and size <= usable_vram_gib - 1.0:
        return "full GPU offload is plausible"
    if size <= system_ram_gib * 0.75:
        return "CPU/GPU split or memory-mapped CPU inference is plausible"
    return "too large for this machine without a different quantization"

fit_table = model_inventory.copy()
fit_table["fit_assessment"] = [fit_assessment(row.size_gib, row.quant_or_format) for row in fit_table.itertuples()]
print(f"Observed system RAM: {system_ram_gib:.1f} GiB")
print(f"Observed GPU VRAM: {gpu_vram_gib:.1f} GiB")
print(f"Planning VRAM after overhead: {usable_vram_gib:.1f} GiB")
fit_table[["file", "quant_or_format", "size_gib", "fit_assessment"]]
Observed system RAM: 31.1 GiB
Observed GPU VRAM: 11.9 GiB
Planning VRAM after overhead: 10.4 GiB
file quant_or_format size_gib fit_assessment
0 gpt-oss-20b-mxfp4.gguf MXFP4 11.28 CPU/GPU split or memory-mapped CPU inference is plausible
1 talkie-1930-13b-it-Q4_K_M.gguf Q4_K_M 7.98 full GPU offload is plausible
2 talkie-Q4_K_M.gguf Q4_K_M 7.87 full GPU offload is plausible
3 Qwen_Qwen3.5-4B-BF16.gguf BF16 7.85 full GPU offload is plausible
4 gemma-4-e4b-it-Q8_0.gguf Q8_0 7.48 full GPU offload is plausible
5 Qwen_Qwen3.5-4B-Q4_K_M.gguf Q4_K_M 2.81 full GPU offload is plausible
6 qwen35_4b_steam_v1_playtime_lora-f16.gguf F16 0.01 full GPU offload is plausible
7 qwen35_4b_rich_history_global_layer24_cvector.gguf adapter/control-vector 0.00 sidecar artifact, not a standalone model 
runtime_checks = pd.DataFrame([
    {"component": "llama-server", "path": str(LLAMA_SERVER), "exists": LLAMA_SERVER.exists(), "executable": os.access(LLAMA_SERVER, os.X_OK)},
    {"component": "llama-cli", "path": str(LLAMA_CLI), "exists": LLAMA_CLI.exists(), "executable": os.access(LLAMA_CLI, os.X_OK)},
    {"component": "llama-bench", "path": str(LLAMA_BENCH), "exists": LLAMA_BENCH.exists(), "executable": os.access(LLAMA_BENCH, os.X_OK)},
    {"component": "nvidia-smi", "path": shutil.which("nvidia-smi"), "exists": shutil.which("nvidia-smi") is not None, "executable": shutil.which("nvidia-smi") is not None},
])
runtime_checks
component path exists executable
0 llama-server /home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-server True True
1 llama-cli /home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-cli True True
2 llama-bench /home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-bench True True
3 nvidia-smi /usr/bin/nvidia-smi True True

The practical conclusion for this workstation is that Qwen 3.5 4B Q4 is the default fast test model, Talkie 13B Q4 is plausible for inference with careful context settings, and full fine-tuning of these generation models is not the consumer-hardware target. Adapter tuning and local serving are the right next steps.

Back to top