diff --git a/ainode/api/server.py b/ainode/api/server.py
index a63da2f..34c9e11 100644
--- a/ainode/api/server.py
+++ b/ainode/api/server.py
@@ -372,6 +372,37 @@ async def _engine_serving(backend, loop) -> bool:
return False
+async def _live_instance_records(manager, loop) -> list:
+ """Probe every managed instance; return the records whose engine answers.
+
+ Also flips each live record's status to ``serving`` (F3). The record is
+ stamped ``starting`` at load time and was never updated once the engine
+ came up, so the announcement advertised a phantom ``starting`` forever and
+ the dashboard kept painting a "STARTING · 8%" progress bar for an instance
+ that was actually serving traffic. The localhost /v1/models probe (via
+ ``_engine_serving``) is the truthful liveness signal, so a passing probe is
+ exactly when the status should read ``serving``.
+ """
+ live = []
+ for inst in manager.instances():
+ if await _engine_serving(inst.backend, loop):
+ if inst.record.status != "serving":
+ inst.record.status = "serving"
+ live.append(inst.record)
+ elif inst.record.status == "serving":
+ # Truthful reset (the other half of F3): the `serving` stamp is a
+ # latch — set once when the engine first answered and, before this,
+ # never cleared. An instance that was serving but whose engine no
+ # longer answers has crashed or been killed out-of-band; leaving the
+ # latch at `serving` makes every consumer that reads `record.status`
+ # without its own probe (e.g. the Server view's LOADED MODELS list)
+ # paint a dead instance as READY forever. Flip it back to `failed`
+ # so status tracks liveness in both directions. If the engine
+ # recovers, the next cycle re-flips it to `serving`.
+ inst.record.status = "failed"
+ return live
+
+
async def _cluster_sync_loop(app: web.Application) -> None:
"""Periodically sync the listener registry into ClusterState."""
try:
@@ -433,11 +464,10 @@ async def _cluster_sync_loop(app: web.Application) -> None:
manager = app.get("instances")
if manager is not None and not manager.is_empty():
# Only advertise instances whose engine actually answers — a dead
- # stacked instance drops out of the broadcast within one cycle.
- live_records = []
- for inst in manager.instances():
- if await _engine_serving(inst.backend, loop):
- live_records.append(inst.record)
+ # stacked instance drops out of the broadcast within one cycle —
+ # and flip each live record's status to `serving` so the UI stops
+ # showing a phantom `starting` progress bar (F3).
+ live_records = await _live_instance_records(manager, loop)
updates["instances"] = [r.to_dict() for r in live_records]
else:
updates["instances"] = (
diff --git a/ainode/api/server_routes.py b/ainode/api/server_routes.py
index ee930c4..9b472f9 100644
--- a/ainode/api/server_routes.py
+++ b/ainode/api/server_routes.py
@@ -198,14 +198,18 @@ async def handle_server_status(request: web.Request) -> web.Response:
web_port = getattr(config, "web_port", 3000)
host = getattr(config, "host", "0.0.0.0")
- # Collect loaded models (local + cluster members)
- local_models = await _probe_loaded_models(session, getattr(config, "api_port", 8000))
+ # Collect loaded models (local primary + local stacked + cluster members).
+ primary_port = getattr(config, "api_port", 8000)
+ local_models = await _probe_loaded_models(session, primary_port)
loaded_models: list[dict] = []
for mid in local_models:
loaded_models.append({
"id": mid,
"node_hostname": config.node_name or "local",
"node_id": config.node_id or "local",
+ "port": primary_port,
+ "ready": True,
+ "ejectable": True,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
@@ -215,6 +219,45 @@ async def handle_server_status(request: web.Request) -> web.Response:
"loaded_at": start_time,
})
+ # Local STACKED instances (2nd+ model on this node, ports 8001+) live in the
+ # InstanceManager, not on the primary vLLM port the probe above hits — so
+ # they were invisible in the Server view (F2). Add a row per stacked
+ # instance. These are local, so the eject endpoint can target them.
+ manager = request.app.get("instances")
+ if manager is not None:
+ try:
+ for inst in manager.instances():
+ rec = inst.record
+ if not rec.model or rec.api_port == primary_port:
+ continue # primary already covered by the probe above
+ # Truthful readiness: probe the stacked instance's own OpenAI
+ # port live rather than trusting rec.status. rec.status is a
+ # latch stamped `serving` when the engine first answered; a
+ # stacked engine that later crashes or is killed out-of-band
+ # keeps reading `serving` (so it would show a green READY badge
+ # and be chat-targetable indefinitely). The /v1/models probe is
+ # the same liveness signal the primary uses two blocks up, so
+ # views and routing agree. The row stays ejectable either way so
+ # a dead instance can still be cleaned up from the UI.
+ stacked_live = await _probe_loaded_models(session, rec.api_port)
+ loaded_models.append({
+ "id": rec.model,
+ "node_hostname": config.node_name or "local",
+ "node_id": config.node_id or "local",
+ "port": rec.api_port,
+ "ready": bool(stacked_live),
+ "ejectable": True,
+ "type": "llm",
+ "format": "SafeTensors",
+ "quantization": None,
+ "size_bytes": 0,
+ "parallel": 1,
+ "capabilities": ["chat", "completions"],
+ "loaded_at": start_time,
+ })
+ except Exception:
+ logger.exception("failed to list local stacked instances")
+
# Include loaded embedding models (in-process, via EmbeddingManager)
embedding_manager = request.app.get("embedding_manager")
if embedding_manager is not None:
@@ -226,6 +269,9 @@ async def handle_server_status(request: web.Request) -> web.Response:
"id": emb["id"],
"node_hostname": hostname,
"node_id": config.node_id or "local",
+ "port": primary_port,
+ "ready": True,
+ "ejectable": True,
"type": "embed",
"format": "SafeTensors",
"quantization": None,
@@ -247,11 +293,43 @@ async def handle_server_status(request: web.Request) -> web.Response:
for m in members:
if m.node_id == config.node_id:
continue
+ member_port = getattr(m, "api_port", 8000) or 8000
+ # Remote instances can't be ejected from here — the eject
+ # endpoint only targets this node's local InstanceManager.
if m.model:
loaded_models.append({
"id": m.model,
"node_hostname": m.node_name,
"node_id": m.node_id,
+ "port": member_port,
+ "ready": True, # model is only broadcast once serving
+ "ejectable": False,
+ "type": "llm",
+ "format": "SafeTensors",
+ "quantization": None,
+ "size_bytes": 0,
+ "parallel": 1,
+ "capabilities": ["chat", "completions"],
+ "loaded_at": getattr(m, "last_seen", start_time),
+ })
+ # Remote STACKED instances (ports 8001+) carried on the peer's
+ # announcement — same source /api/nodes exposes (F2).
+ for inst in (getattr(m, "instances", []) or []):
+ if not isinstance(inst, dict):
+ continue
+ im = inst.get("model")
+ if not im:
+ continue
+ inst_port = inst.get("api_port") or member_port
+ if im == m.model and inst_port == member_port:
+ continue # primary already added above
+ loaded_models.append({
+ "id": im,
+ "node_hostname": m.node_name,
+ "node_id": m.node_id,
+ "port": inst_port,
+ "ready": inst.get("status") == "serving",
+ "ejectable": False,
"type": "llm",
"format": "SafeTensors",
"quantization": None,
@@ -261,7 +339,7 @@ async def handle_server_status(request: web.Request) -> web.Response:
"loaded_at": getattr(m, "last_seen", start_time),
})
except Exception:
- pass
+ logger.exception("failed to list cluster-member instances")
# Request counters
counters = request.app.get("api_log_counters") or {}
diff --git a/ainode/web/static/js/app.js b/ainode/web/static/js/app.js
index f0eafcb..784ec30 100644
--- a/ainode/web/static/js/app.js
+++ b/ainode/web/static/js/app.js
@@ -518,16 +518,30 @@ const AINode = {
badge.innerHTML = '⬆ Update available: v' + info.latest;
badge.title = 'Click to update to v' + info.latest;
badge.addEventListener('click', function () {
- if (!confirm('Update AINode to v' + info.latest + '? The service will restart.')) return;
+ // Fleet-wide update: POST /api/cluster/update-all (resolves one target tag
+ // and threads it to every peer, master last) instead of the head-only
+ // /api/engine/update. F4.
+ var nodeCount = (self.state.nodes || []).length || 1;
+ var nodeWord = nodeCount === 1 ? 'node' : 'nodes';
+ if (!confirm('Update all ' + nodeCount + ' ' + nodeWord + ' to v' + info.latest + '? Services restart one by one.')) return;
badge.textContent = 'Updating...';
badge.disabled = true;
- fetch('/api/engine/update', { method: 'POST' })
+ fetch('/api/cluster/update-all', { method: 'POST' })
.then(function (r) { return r.json(); })
- .then(function () {
- self.toast('Update started — service will restart in a moment', 'success');
- badge.textContent = 'Restarting...';
- // Re-check version in 60 seconds
- setTimeout(function () { self.checkVersion(); }, 60000);
+ .then(function (data) {
+ if (data && data.error) {
+ badge.textContent = '⬆ Update available: v' + info.latest;
+ badge.disabled = false;
+ self.toast('Update failed: ' + data.error, 'error');
+ return;
+ }
+ self.toast('Fleet update started — nodes restart one by one', 'success');
+ badge.textContent = 'Updating fleet...';
+ // Surface live per-node progress in the cluster panel if it's mounted.
+ if (data && data.update_id) self._pollClusterUpdate(data.update_id);
+ // Re-check version in 90 seconds (bumped from 60s — a rolling fleet
+ // restart takes longer than a single-node one).
+ setTimeout(function () { self.checkVersion(); }, 90000);
})
.catch(function () {
badge.textContent = '⬆ Update available: v' + info.latest;
@@ -710,7 +724,10 @@ const AINode = {
// the federated /v1/models the chat dropdown uses. Skip the distributed one.
var seen = {};
(this.state.nodes || []).forEach(function (n) {
- var host = n.hostname || n.node_id;
+ // Show the friendly node NAME (Spark-2-DGX), not the raw node-id hex.
+ // node_name is the id→name mapping carried on every /api/nodes entry;
+ // fall back to hostname, then the id, when the name is unknown (F1).
+ var host = n.node_name || n.hostname || n.node_id;
var modelName = n.model;
if (modelName && !(distModel && modelName === distModel)) {
var key = modelName + '@' + (n.node_id || n.hostname);
@@ -739,7 +756,13 @@ const AINode = {
model: im,
strategy: 'stacked',
nodes: [host + (inst.api_port ? ':' + inst.api_port : '')],
- status: (inst.status === 'serving' || n.engine_ready) ? 'READY' : 'STARTING',
+ // Use the stacked instance's OWN status, not the node's primary
+ // readiness. n.engine_ready reflects the PRIMARY engine (and for the
+ // local node is hardcoded online → always true), so OR-ing it in made
+ // a still-loading or dead stacked model read READY the moment the
+ // primary was up. inst.status is the per-instance truth (the head
+ // only advertises live instances, flipped to `serving`).
+ status: inst.status === 'serving' ? 'READY' : 'STARTING',
badge: 'STACKED' + (inst.api_port ? ' · :' + inst.api_port : ''),
});
});
@@ -5278,8 +5301,15 @@ const AINode = {
_renderLoadedCard(m, idx) {
var id = m.id || 'unknown';
var nodeHost = m.node_hostname || m.node_id || 'local';
+ // Stacked instances live on ports 8001+ — show the port so two models on
+ // one node are distinguishable (F2). Primary instances omit it.
+ var portStr = (m.port && m.port !== 8000) ? ' · :' + m.port : '';
var type = m.type || 'llm';
var isEmbed = type === 'embed';
+ var ready = m.ready !== false;
+ // Remote instances (loaded on a peer) can't be ejected from here — the eject
+ // endpoint only targets this node's local InstanceManager (F2).
+ var ejectable = m.ejectable !== false;
var sizeStr = m.size_bytes > 0 ? this.formatBytes(m.size_bytes) : '—';
var parallel = m.parallel || 1;
var selected = (this._serverState.selectedModelId === id) ? ' selected' : '';
@@ -5292,10 +5322,16 @@ const AINode = {
var primaryIconBtn = isEmbed
? ' '
: ' ';
+ var statusBadge = ready
+ ? 'READY'
+ : 'STARTING';
+ var ejectBtn = ejectable
+ ? ' '
+ : '';
return '