diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 52c5de7b..6b90627a 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -62,6 +62,8 @@ Emitted when a manually added node has been probed and its initial status determ Each node is probed for both inference engines: Ollama on its default `:11434` (`GET /` + `/api/tags`) and LM Studio on its default `:1234` (`GET /v1/models`, which doubles as the liveness check and the model list). `lmstudio_up` / `lmstudio_port` / `lmstudio_models` mirror the `ollama_*` fields and let a supervising broker bridge the node into that engine's `nvpair-proxy` instance the same way it bridges Ollama into its own. A node can run either engine, both, or neither. +When nothing answers on `:11434`, the same Ollama probe is retried on `:17434`, the default port of [llmman](https://github.com/llmmanorg/llmman), which serves the Ollama API. Such a node reports `ollama_up: true` with `ollama_port: 17434` and is bridged into `ollama-proxy` like an Ollama node. + ### `node/updated` Emitted when a periodic probe detects a change (service going up/down, models, @@ -134,13 +136,13 @@ Changes the log level at runtime. Accepted as either a request (answered with `{ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, for: -- **Ollama** on port 11434: health check (`GET /`) and model list (`GET /api/tags`) +- **Ollama** on port 11434: health check (`GET /`) and model list (`GET /api/tags`). If it does not answer, the same probe is sent to port 17434, where [llmman](https://github.com/llmmanorg/llmman) serves the Ollama API; `ollama_port` reports whichever port answered - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list - **Node Info** on port 14318, or `tls_port` over HTTPS: hardware inventory and identity (`GET /v1/node-info`) A node can have any combination of these, or none if the target is unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. -The three engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. +The engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. ## Shutdown diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 55a4040a..4b238d82 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -251,7 +251,7 @@ func (m *Manager) probeNode(entry ManualEntry) { addr := entry.Address id := nodeID(entry) - ollamaUp, ollamaModels := m.probeOllama(addr, 11434) + ollamaUp, ollamaAPIPort, ollamaModels := m.probeOllamaAPI(addr) lmStudioUp, lmStudioModels := m.probeLMStudio(addr, lmStudioPort) // Pick scheme + port + client based on the entry's TLS hint. @@ -285,7 +285,7 @@ func (m *Manager) probeNode(entry ManualEntry) { Name: entry.Name, Address: addr, OllamaUp: ollamaUp, - OllamaPort: 11434, + OllamaPort: ollamaAPIPort, OllamaModels: ollamaModels, LMStudioUp: lmStudioUp, LMStudioPort: lmStudioPort, @@ -403,6 +403,24 @@ func probeFailedID(nodeID string) string { // manager (which only governs the local engine). const lmStudioPort = 1234 +// Default ports of Ollama and of llmman (https://github.com/llmmanorg/llmman), +// which serves the same Ollama API on 17434. +const ( + ollamaPort = 11434 + llmmanPort = 17434 +) + +// probeOllamaAPI probes Ollama first, then llmman, and returns whether one +// answered, its port (ollamaPort when neither did) and its models. +func (m *Manager) probeOllamaAPI(addr string) (bool, int, []string) { + for _, port := range []int{ollamaPort, llmmanPort} { + if up, models := m.probeOllama(addr, port); up { + return true, port, models + } + } + return false, ollamaPort, nil +} + // probeLMStudio checks LM Studio's OpenAI-compatible server on addr:port. A // single GET /v1/models doubles as the liveness check and the model list (the // response is {"data":[{"id":"..."}],...}). Returns whether it is up and the @@ -539,7 +557,7 @@ func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { ID: id, Name: entry.Name, Address: entry.Address, - OllamaPort: 11434, + OllamaPort: ollamaPort, NodeInfoPort: nodeInfoPort, TLSEnabled: entry.TLSPort > 0, MTLSRequired: entry.TLSPort > 0 && entry.MTLS, diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index 30c20b56..4e72657e 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -11,6 +11,7 @@ import ( "io" "net" "net/http" + "strconv" "strings" "sync" "testing" @@ -88,7 +89,20 @@ func newTestManager() (*Manager, *captureRW, *fakeRoundTripper) { func configureHealthyNode(t *testing.T, rt *fakeRoundTripper, addr string, models []string, info NodeInfoResponse) { t.Helper() - host := net.JoinHostPort(addr, "11434") + configureHealthyOllamaAPI(t, rt, addr, ollamaPort, models) + + infoHost := net.JoinHostPort(addr, "14318") + rt.set(http.MethodGet, infoHost, "/v1/node-info", func(*http.Request) (*http.Response, error) { + data, err := json.Marshal(info) + assert.NoError(t, err, "marshal node info") + return httpJSON(http.StatusOK, string(data)) + }) +} + +// configureHealthyOllamaAPI registers GET / and GET /api/tags on addr:port. +func configureHealthyOllamaAPI(t *testing.T, rt *fakeRoundTripper, addr string, port int, models []string) { + t.Helper() + host := net.JoinHostPort(addr, strconv.Itoa(port)) rt.set(http.MethodGet, host, "/", func(*http.Request) (*http.Response, error) { return httpJSON(http.StatusOK, `{}`) }) @@ -107,13 +121,6 @@ func configureHealthyNode(t *testing.T, rt *fakeRoundTripper, addr string, model assert.NoError(t, err, "marshal model inventory") return httpJSON(http.StatusOK, string(data)) }) - - infoHost := net.JoinHostPort(addr, "14318") - rt.set(http.MethodGet, infoHost, "/v1/node-info", func(*http.Request) (*http.Response, error) { - data, err := json.Marshal(info) - assert.NoError(t, err, "marshal node info") - return httpJSON(http.StatusOK, string(data)) - }) } // configureHealthyLMStudio registers a 200 GET /v1/models on addr:1234 @@ -157,6 +164,35 @@ func TestProbeLMStudio_ReportsModels(t *testing.T) { assert.Nil(t, downModels, "expected absent lmstudio down, got up (%v, %v)", downUp, downModels) } +// TestProbeOllamaAPI_FallsBackToLLMMan: Ollama wins when both answer, llmman is +// reported with its port when only it answers, neither reports down on 11434. +func TestProbeOllamaAPI_FallsBackToLLMMan(t *testing.T) { + m, _, rt := newTestManager() + configureHealthyOllamaAPI(t, rt, "both.local", ollamaPort, []string{"llama3"}) + configureHealthyOllamaAPI(t, rt, "both.local", llmmanPort, []string{"gemma4"}) + configureHealthyOllamaAPI(t, rt, "llmman.local", llmmanPort, []string{"gemma4", "qwen3.8"}) + + tests := []struct { + name string + addr string + wantUp bool + wantPort int + wantModels []string + }{ + {"both answer", "both.local", true, ollamaPort, []string{"llama3"}}, + {"only llmman", "llmman.local", true, llmmanPort, []string{"gemma4", "qwen3.8"}}, + {"neither", "absent.local", false, ollamaPort, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + up, port, models := m.probeOllamaAPI(tc.addr) + assert.Equal(t, tc.wantUp, up, "up") + assert.Equal(t, tc.wantPort, port, "port") + assert.Equal(t, tc.wantModels, models, "models") + }) + } +} + func requestMessage(t *testing.T, id int, method string, params any) *Message { t.Helper() idData, err := json.Marshal(id)