354 sor
15 KiB
Python
Executable File
354 sor
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Controlla se un repo Hugging Face e' plausibilmente compatibile con questo
|
|
cluster vLLM/ROCm PRIMA di scaricarlo: dimensione totale, architettura
|
|
supportata da vLLM, formato/quantizzazione e rischi noti.
|
|
|
|
Uso:
|
|
./check-model.py <repo_id> [--node-ram-gb 128] [--gpu-util 0.75]
|
|
[--kv-reserve-gb 15] [--cluster] [--worker-ram-gb 96]
|
|
|
|
Nota: e' un controllo euristico basato sui metadati pubblici del repo,
|
|
non sostituisce un test reale (es. compatibilita' dei kernel come Marlin
|
|
per AWQ va comunque verificata avviando vllm serve).
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import urllib.request
|
|
|
|
HF_API = "https://huggingface.co/api/models/{repo_id}"
|
|
VLLM_REGISTRY_URL = (
|
|
"https://raw.githubusercontent.com/vllm-project/vllm/main/"
|
|
"vllm/model_executor/models/registry.py"
|
|
)
|
|
|
|
|
|
def fetch_json(url: str):
|
|
req = urllib.request.Request(url, headers={"User-Agent": "check-model.py"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return json.load(resp)
|
|
|
|
|
|
def fetch_text(url: str) -> str:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "check-model.py"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read().decode("utf-8")
|
|
|
|
|
|
def fetch_vllm_supported_architectures() -> set[str]:
|
|
"""Estrae le chiavi (nomi architettura HF) dai dict *_MODELS in registry.py."""
|
|
text = fetch_vllm_registry_text()
|
|
archs: set[str] = set()
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if line.startswith('"') and '": (' in line:
|
|
key = line.split('"')[1]
|
|
archs.add(key)
|
|
return archs
|
|
|
|
|
|
_REGISTRY_TEXT_CACHE: str | None = None
|
|
|
|
|
|
def fetch_vllm_registry_text() -> str:
|
|
global _REGISTRY_TEXT_CACHE
|
|
if _REGISTRY_TEXT_CACHE is None:
|
|
_REGISTRY_TEXT_CACHE = fetch_text(VLLM_REGISTRY_URL)
|
|
return _REGISTRY_TEXT_CACHE
|
|
|
|
|
|
GITHUB_TREE_URL = "https://api.github.com/repos/vllm-project/vllm/git/trees/main?recursive=1"
|
|
_GITHUB_TREE_CACHE: dict | None = None
|
|
_PARSER_SUFFIXES = (
|
|
"_engine_tool_parser", "_tool_parser",
|
|
"_engine_reasoning_parser", "_reasoning_parser",
|
|
)
|
|
|
|
|
|
def fetch_github_tree() -> dict:
|
|
global _GITHUB_TREE_CACHE
|
|
if _GITHUB_TREE_CACHE is None:
|
|
_GITHUB_TREE_CACHE = fetch_json(GITHUB_TREE_URL)
|
|
return _GITHUB_TREE_CACHE
|
|
|
|
|
|
def parser_base_names(tree: dict, prefix: str) -> set[str]:
|
|
"""Nomi base dei parser (tool/reasoning) ricavati dai file sorgente in un dato prefisso."""
|
|
names: set[str] = set()
|
|
for entry in tree.get("tree", []):
|
|
path = entry.get("path", "")
|
|
if not path.startswith(prefix) or not path.endswith(".py"):
|
|
continue
|
|
base = path[len(prefix):-3]
|
|
if base in ("__init__", "utils", "abstract_tool_parser", "streaming", "structural_tag_registry"):
|
|
continue
|
|
for suf in _PARSER_SUFFIXES:
|
|
if base.endswith(suf):
|
|
base = base[: -len(suf)]
|
|
break
|
|
names.add(base)
|
|
return names
|
|
|
|
|
|
def guess_parser_candidates(names: set[str], haystack_strings: list[str]) -> list[str]:
|
|
"""Euristica: confronta i nomi base dei parser con stringhe note del repo (model_type,
|
|
architetture, tag, base_model...), solo alfanumerico, no simboli."""
|
|
def norm(s: str) -> str:
|
|
return "".join(ch for ch in s.lower() if ch.isalnum())
|
|
|
|
haystacks = [norm(s) for s in haystack_strings]
|
|
candidates = []
|
|
for name in sorted(names):
|
|
n = norm(name)
|
|
if not n:
|
|
continue
|
|
if any(n in h or h.startswith(n) for h in haystacks if h):
|
|
candidates.append(name)
|
|
return sorted(candidates)
|
|
|
|
|
|
def find_tool_chat_templates(tree: dict, candidates: list[str]) -> list[str]:
|
|
"""Cerca in examples/tool_chat_template_*.jinja un file che corrisponda a uno dei candidati."""
|
|
found = []
|
|
for entry in tree.get("tree", []):
|
|
path = entry.get("path", "")
|
|
if not path.startswith("examples/tool_chat_template_") or not path.endswith(".jinja"):
|
|
continue
|
|
base = path[len("examples/tool_chat_template_"):-len(".jinja")]
|
|
base_norm = "".join(ch for ch in base.lower() if ch.isalnum())
|
|
for c in candidates:
|
|
c_norm = "".join(ch for ch in c.lower() if ch.isalnum())
|
|
if c_norm and (c_norm in base_norm or base_norm in c_norm):
|
|
found.append(path)
|
|
break
|
|
return found
|
|
|
|
|
|
def human_gb(num_bytes: float) -> str:
|
|
return f"{num_bytes / 1e9:.1f} GB"
|
|
|
|
|
|
GREEN = "\033[32m"
|
|
YELLOW = "\033[33m"
|
|
RED = "\033[31m"
|
|
RESET = "\033[0m"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("repo_id", help="es. cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit")
|
|
ap.add_argument("--node-ram-gb", type=float, default=128.0, help="RAM del nodo per il controllo singolo nodo (default: 128, l'head)")
|
|
ap.add_argument("--worker-ram-gb", type=float, default=96.0, help="RAM del worker, nodo piu' piccolo che vincola il budget in modalita' --cluster (default: 96)")
|
|
ap.add_argument("--gpu-util", type=float, default=0.75, help="GPU_MEMORY_UTILIZATION da usare nel calcolo (default: 0.75)")
|
|
ap.add_argument("--kv-reserve-gb", type=float, default=15.0, help="GB minimi da lasciare liberi per la KV cache (default: 15)")
|
|
ap.add_argument("--cluster", action="store_true", help="Valuta per il cluster a 2 nodi (TP=2) invece che singolo nodo")
|
|
args = ap.parse_args()
|
|
|
|
print(f"== {args.repo_id} ==\n")
|
|
|
|
suggestions: list[tuple[str, str]] = [("MODEL_PATH", args.repo_id)]
|
|
|
|
try:
|
|
meta = fetch_json(HF_API.format(repo_id=args.repo_id))
|
|
except Exception as e:
|
|
print(f"ERRORE: impossibile leggere i metadati da Hugging Face: {e}")
|
|
return 1
|
|
|
|
if meta.get("gated"):
|
|
print("ATTENZIONE: repo gated, servira' HF_TOKEN con accesso per scaricarlo.\n")
|
|
|
|
siblings = [s.get("rfilename", "") for s in meta.get("siblings", [])]
|
|
gguf_files = [f for f in siblings if f.lower().endswith(".gguf")]
|
|
|
|
tags = meta.get("tags") or []
|
|
base_models = [
|
|
t.split(":", 1)[1]
|
|
for t in tags
|
|
if t.startswith("base_model:") and not t.startswith("base_model:quantized:")
|
|
]
|
|
if base_models:
|
|
print("Repo originale (base_model):", ", ".join(base_models))
|
|
if gguf_files:
|
|
print(f" -> se e' un modello multimodale, serve --tokenizer {base_models[0]} (i GGUF multimodali non includono il tokenizer originale)")
|
|
suggestions.append(("TOKENIZER", f"{base_models[0]} # solo se multimodale"))
|
|
print()
|
|
|
|
config = meta.get("config") or {}
|
|
architectures = config.get("architectures") or []
|
|
model_type = config.get("model_type", "")
|
|
quant_config = config.get("quantization_config") or {}
|
|
|
|
# --- Architettura supportata da vLLM? ---
|
|
if architectures:
|
|
try:
|
|
supported = fetch_vllm_supported_architectures()
|
|
except Exception as e:
|
|
supported = set()
|
|
print(f"(impossibile verificare la lista architetture vLLM: {e})")
|
|
print("Architettura:", ", ".join(architectures))
|
|
for arch in architectures:
|
|
if not supported:
|
|
continue
|
|
if arch in supported:
|
|
print(f" -> OK, '{arch}' e' nel registry di vLLM")
|
|
else:
|
|
print(f" -> ATTENZIONE: '{arch}' NON risulta nel registry di vLLM (potrebbe non essere supportata)")
|
|
else:
|
|
print("Architettura: non trovata nei metadati (repo solo-GGUF? controlla il README)")
|
|
|
|
if model_type:
|
|
print("model_type:", model_type)
|
|
|
|
is_moe_hint = "moe" in model_type.lower() or any("moe" in a.lower() for a in architectures)
|
|
print("MoE (euristico):", "si'" if is_moe_hint else "probabilmente no")
|
|
print()
|
|
|
|
# --- Suggerimenti tool-call-parser / reasoning-parser / chat-template ---
|
|
# Usa tutto cio' che puo' indicare la famiglia del modello: utile anche per i repo
|
|
# solo-GGUF che non espongono model_type/architectures (es. tag "minimax_m2" o
|
|
# base_model "MiniMaxAI/MiniMax-M2.7").
|
|
haystack_strings = [model_type, *architectures, *tags, *base_models]
|
|
if any(haystack_strings):
|
|
try:
|
|
gh_tree = fetch_github_tree()
|
|
tool_names = parser_base_names(gh_tree, "vllm/tool_parsers/")
|
|
reasoning_names = parser_base_names(gh_tree, "vllm/reasoning/")
|
|
tool_candidates = guess_parser_candidates(tool_names, haystack_strings)
|
|
reasoning_candidates = guess_parser_candidates(reasoning_names, haystack_strings)
|
|
if tool_candidates:
|
|
print(f"Possibili --tool-call-parser: {', '.join(tool_candidates)} (verifica avviando vllm serve)")
|
|
suggestions.append(("ENABLE_AUTO_TOOL_CHOICE", "1"))
|
|
suggestions.append(("TOOL_CALL_PARSER", " o ".join(tool_candidates) + " # verifica"))
|
|
if reasoning_candidates:
|
|
print(f"Possibili --reasoning-parser: {', '.join(reasoning_candidates)} (verifica avviando vllm serve)")
|
|
suggestions.append(("REASONING_PARSER", " o ".join(reasoning_candidates) + " # verifica"))
|
|
templates = find_tool_chat_templates(gh_tree, tool_candidates + reasoning_candidates)
|
|
if templates:
|
|
print(f"Possibile chat template dedicato nel repo vLLM (NON incluso nella nostra immagine): {', '.join(templates)}")
|
|
suggestions.append(("CHAT_TEMPLATE", " o ".join(templates) + " # non incluso, da scaricare"))
|
|
if not tool_candidates and not reasoning_candidates:
|
|
print("Nessun tool/reasoning parser candidato trovato per euristica (nome architettura troppo diverso dai parser noti)")
|
|
except Exception as e:
|
|
print(f"(impossibile suggerire tool/reasoning parser: {e})")
|
|
print()
|
|
|
|
# --- Quantizzazione ---
|
|
if quant_config:
|
|
method = quant_config.get("quant_method", "?")
|
|
bits = quant_config.get("bits", "?")
|
|
group_size = quant_config.get("group_size")
|
|
print(f"Quantizzazione: {method}, {bits} bit, group_size={group_size}")
|
|
if method in ("awq", "gptq") and is_moe_hint:
|
|
print(f" -> {YELLOW}CERTO{RESET}: il kernel Marlin veloce per MoE e' disabilitato in modo incondizionato su ROCm")
|
|
print(f" (vllm/.../marlin_utils.py: 'if current_platform.is_rocm(): return False', a prescindere da group_size/shape)")
|
|
print(" Fallback garantito a kernel WNA16 (piu' lento) per TUTTI i layer MoE, qualunque sia il group_size.")
|
|
elif gguf_files:
|
|
print("Quantizzazione: repo GGUF, vedi varianti sotto (nome file di solito indica il quant, es. Q4_K_M)")
|
|
else:
|
|
print("Quantizzazione: nessuna rilevata nei metadati (probabilmente pesi non quantizzati / fp16-bf16)")
|
|
print()
|
|
|
|
# --- Dimensione ---
|
|
used_storage = meta.get("usedStorage")
|
|
safetensors_total_params = (meta.get("safetensors") or {}).get("total")
|
|
|
|
if args.cluster:
|
|
worker_budget = args.worker_ram_gb * args.gpu_util - args.kv_reserve_gb
|
|
budget_gb = worker_budget * 2 # split simmetrico TP=2, vincolato dal nodo piu' piccolo
|
|
budget_desc = f"cluster (TP=2, vincolato dal worker da {args.worker_ram_gb:.0f}GB)"
|
|
else:
|
|
budget_gb = args.node_ram_gb * args.gpu_util - args.kv_reserve_gb
|
|
budget_desc = f"singolo nodo ({args.node_ram_gb:.0f}GB, util={args.gpu_util})"
|
|
|
|
def rank(total_gb: float) -> int:
|
|
"""0 = verde/OK, 1 = giallo/a malapena, 2 = rosso/non ci sta."""
|
|
margin = budget_gb - total_gb
|
|
if margin < 0:
|
|
return 2
|
|
if margin < args.kv_reserve_gb:
|
|
return 1
|
|
return 0
|
|
|
|
def verdict(total_gb: float) -> str:
|
|
margin = budget_gb - total_gb
|
|
if margin < 0:
|
|
return f"{RED}NON CI STA (supera il budget di {-margin:.0f} GB){RESET}"
|
|
if margin < args.kv_reserve_gb:
|
|
return f"{YELLOW}CI STA A MALAPENA (solo {margin:.0f} GB di margine oltre la riserva KV cache){RESET}"
|
|
return f"{GREEN}OK ({margin:.0f} GB di margine oltre la riserva KV cache){RESET}"
|
|
|
|
def status_badge(r: int) -> str:
|
|
if r == 0:
|
|
return f"{GREEN}OK{RESET}"
|
|
if r == 1:
|
|
return f"{YELLOW}A MALAPENA{RESET}"
|
|
return f"{RED}NO{RESET}"
|
|
|
|
def margin_str(total_gb: float) -> str:
|
|
margin = budget_gb - total_gb
|
|
sign = "+" if margin >= 0 else ""
|
|
return f"{sign}{margin:.0f} GB"
|
|
|
|
print(f"Budget {budget_desc}: ~{budget_gb:.0f} GB\n")
|
|
|
|
if gguf_files:
|
|
try:
|
|
tree = fetch_json(HF_API.format(repo_id=args.repo_id) + "/tree/main?recursive=true")
|
|
sizes = {t["path"]: t.get("size", 0) for t in tree if t.get("type") == "file"}
|
|
except Exception:
|
|
sizes = {}
|
|
# Raggruppa per "famiglia" di quant (rimuove i suffissi -00001-of-0000N)
|
|
import re
|
|
groups: dict[str, int] = {}
|
|
for f in gguf_files:
|
|
key = re.sub(r"-\d{5}-of-\d{5}", "", f)
|
|
groups[key] = groups.get(key, 0) + sizes.get(f, 0)
|
|
known = [(key, total) for key, total in groups.items() if total]
|
|
unknown = [key for key, total in groups.items() if not total]
|
|
# Verde (piu' piccolo -> piu' grande), separatore, giallo, separatore, rosso.
|
|
known.sort(key=lambda kt: (rank(kt[1] / 1e9), kt[1]))
|
|
|
|
all_names = [k for k, _ in known] + unknown
|
|
name_w = max([len("VARIANTE")] + [len(n) for n in all_names])
|
|
size_w = max([len("DIMENSIONE")] + [len(human_gb(t)) for _, t in known])
|
|
margin_w = max([len("MARGINE")] + [len(margin_str(t / 1e9)) for _, t in known])
|
|
|
|
print("File GGUF trovati:")
|
|
print(f" {'VARIANTE':<{name_w}} {'DIMENSIONE':>{size_w}} {'MARGINE':>{margin_w}} STATO")
|
|
print(f" {'-' * name_w} {'-' * size_w} {'-' * margin_w} -----")
|
|
last_rank = None
|
|
for key, total in known:
|
|
r = rank(total / 1e9)
|
|
if last_rank is not None and r != last_rank:
|
|
print(f" {'-' * name_w} {'-' * size_w} {'-' * margin_w} -----")
|
|
print(f" {key:<{name_w}} {human_gb(total):>{size_w}} {margin_str(total / 1e9):>{margin_w}} {status_badge(r)}")
|
|
last_rank = r
|
|
for key in sorted(unknown):
|
|
print(f" {key:<{name_w}} {'?':>{size_w}} {'?':>{margin_w}} ?")
|
|
else:
|
|
total_bytes = used_storage
|
|
if total_bytes:
|
|
print(f"Dimensione totale repo (usedStorage): {human_gb(total_bytes)}")
|
|
if safetensors_total_params:
|
|
print(f"Parametri totali (safetensors): {safetensors_total_params / 1e9:.1f} B")
|
|
if total_bytes:
|
|
print(f"-> {verdict(total_bytes / 1e9)}")
|
|
else:
|
|
print("Dimensione non determinabile automaticamente.")
|
|
|
|
print()
|
|
print("Variabili quadlet suggerite:")
|
|
var_w = max(len("VARIABILE"), max(len(v) for v, _ in suggestions))
|
|
val_w = max(len("VALORE"), max(len(v) for _, v in suggestions))
|
|
print(f" {'VARIABILE':<{var_w}} VALORE")
|
|
print(f" {'-' * var_w} {'-' * min(val_w, 60)}")
|
|
for name, value in suggestions:
|
|
print(f" {name:<{var_w}} {value}")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|