diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..552f221 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +*.log diff --git a/README.md b/README.md index 58848aa..0b08c37 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,187 @@ # shre-embed -Shared embedding + semantic search service for the estate (Qdrant + local Ollama) \ No newline at end of file +Shared embedding + semantic search service for the estate. Plain Node.js + +Express, ESM, no build step. Embeds text via a local **Ollama** instance and +stores/searches vectors in **Qdrant**. + +## Design principle: three-valued observability + +Every probe and every upstream-dependent answer in this service distinguishes +**three** states, never two: + +| state | meaning | +|---|---| +| `ok` / present | the probe saw the thing and it works / exists | +| `absent` | the probe **saw the surface** and the thing is genuinely missing (e.g. Ollama reachable but the embed model is not installed) | +| `unobservable` | the probe **could not see**: dependency down, timeout, auth failure, endpoint missing, HTTP 5xx from the surface | + +`unobservable` is **never** collapsed into "absent", "zero results" or +"healthy". Every result carries its `measurement_surface` (the exact URL that +was measured) and `observed_at` (ISO timestamp). This exists to prevent a +recorded estate incident class: healthchecks probing `/` and reporting healthy +while `/api` was 502, and an MCP tool answering `0` for data that was never +collectable. + +Concretely: + +- `/health` returns overall `"ok"` only when **all** dependencies are `ok`; + otherwise `"degraded"` with HTTP 503. It never reports plain healthy while a + dependency is unobservable. +- `/v1/embed`, `/v1/index`, `/v1/search` return HTTP 502 with + `{"status":"unobservable", "measurement_surface": ..., "observed_at": ...}` + when an upstream call fails — a failed search is never presented as an empty + result set, and a failed index is never presented as `indexed: 0`. + +## Configuration (env, with defaults) + +| var | default | notes | +|---|---|---| +| `PORT` | `5499` | HTTP listen port (binds 127.0.0.1) | +| `QDRANT_URL` | `http://127.0.0.1:6333` | Qdrant REST endpoint | +| `OLLAMA_URL` | `http://127.0.0.1:11436` | **11436, not 11434.** On this Mac, `127.0.0.1:11434` is a known-broken Colima SSH-mux forward that accepts connections (tags respond) but hangs on generation/embedding. Native Ollama listens on `11436`. | +| `EMBED_MODEL` | `nomic-embed-text` | Ollama embedding model | +| `PROBE_TIMEOUT_MS` | `2500` | timeout for `/health` dependency probes | +| `UPSTREAM_TIMEOUT_MS` | `30000` | timeout for embed/index/search upstream calls | + +## Run + +```sh +npm install +npm start # listens on 127.0.0.1:5499 +npm test # node:test unit tests; no Qdrant/Ollama required +``` + +## Endpoints + +### `GET /health` + +Three-valued, per-dependency health with exact probed surfaces. + +```json +{ + "status": "degraded", + "service": "shre-embed", + "observed_at": "2026-08-22T15:04:05.000Z", + "dependencies": { + "qdrant": { + "status": "ok", + "measurement_surface": "http://127.0.0.1:6333/readyz", + "observed_at": "2026-08-22T15:04:05.000Z", + "http_status": 200, + "latency_ms": 3 + }, + "ollama": { + "status": "unobservable", + "measurement_surface": "http://127.0.0.1:11436/api/tags", + "observed_at": "2026-08-22T15:04:05.000Z", + "reason": "fetch failed: ECONNREFUSED: fetch failed", + "latency_ms": 1 + }, + "embed_model": { + "status": "unobservable", + "measurement_surface": "http://127.0.0.1:11436/api/tags", + "observed_at": "2026-08-22T15:04:05.000Z", + "model": "nomic-embed-text", + "reason": "ollama unobservable, model presence not measurable" + } + } +} +``` + +- HTTP 200 when overall `"ok"`, HTTP 503 when `"degraded"`. +- `embed_model` demonstrates all three values: `ok` (installed), `absent` + (Ollama reachable but model not pulled — a true gap), `unobservable` + (Ollama itself could not be seen, so the model's presence is unknowable). + +### `POST /v1/embed` + +```json +{ "texts": ["hello", "world"] } +``` + +→ `200 {"vectors": [[...],[...]], "model": "nomic-embed-text", "measurement_surface": "http://127.0.0.1:11436/api/embeddings", "observed_at": "..."}` + +Each text is embedded via `POST {OLLAMA_URL}/api/embeddings` with +`{"model": EMBED_MODEL, "prompt": text}`. + +### `POST /v1/index` + +```json +{ + "collection": "notes", + "items": [ + { "id": 1, "text": "alpha", "payload": { "src": "obsidian" } }, + { "id": 2, "text": "beta" } + ] +} +``` + +Embeds every `text`, auto-creates the Qdrant collection if missing (Cosine +distance, vector size taken from the first embedding), and upserts points with +`payload` (the original `text` is folded into the payload as `text`). + +→ `200 {"indexed": 2, "collection": "notes", "qdrant": {...}, "measurement_surface": ..., "observed_at": ...}` + +### `POST /v1/search` + +```json +{ "collection": "notes", "query": "find alpha", "limit": 5 } +``` + +Embeds the query, then `POST {QDRANT_URL}/collections/{collection}/points/search` +with `with_payload: true`. `limit` defaults to 10, capped at 100. + +→ `200 {"results": [{"id": "a", "score": 0.91, "payload": {...}}, ...], "collection": ..., "limit": ..., "measurement_surface": ..., "observed_at": ...}` + +### Error shape for all `/v1/*` endpoints + +- `400 {"error": "..."}` — invalid request body. +- `502 {"error": "...", "status": "unobservable", "measurement_surface": "", "observed_at": "...", "upstream_http_status": 500}` — + the upstream could not be observed; the truthful answer is "unknown", never + "empty" or "zero". + +## Example launchd plist (documentation only — do NOT install blindly) + +The estate convention is `ai.shre.` labels. This is a reference +template only; deployment goes through the normal ops-from-git flow, not by +hand-installing this file. + +```xml + + + + + Labelai.shre.embed + ProgramArguments + + /usr/local/bin/node + /opt/shre/shre-embed/src/server.js + + WorkingDirectory/opt/shre/shre-embed + EnvironmentVariables + + PORT5499 + QDRANT_URLhttp://127.0.0.1:6333 + + OLLAMA_URLhttp://127.0.0.1:11436 + EMBED_MODELnomic-embed-text + + RunAtLoad + KeepAlive + StandardOutPath/tmp/ai.shre.embed.out.log + StandardErrorPath/tmp/ai.shre.embed.err.log + + +``` + +## Layout + +``` +src/config.js env config with estate-safe defaults +src/probe.js three-valued probe primitives (ok | absent | unobservable) +src/upstreams.js Ollama + Qdrant clients (injectable fetch) +src/app.js express app factory (createApp(config, {fetchImpl})) +src/server.js entrypoint +test/app.test.js node:test unit tests, run with zero live dependencies +``` diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..2777d9d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,831 @@ +{ + "name": "shre-embed", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "shre-embed", + "version": "0.1.0", + "dependencies": { + "express": "^4.21.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.6", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", + "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.15.1", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..d317e12 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "shre-embed", + "version": "0.1.0", + "description": "Shared embedding + semantic search service for the estate (Qdrant + local Ollama)", + "type": "module", + "main": "src/server.js", + "scripts": { + "start": "node src/server.js", + "test": "node --test" + }, + "engines": { + "node": ">=20" + }, + "dependencies": { + "express": "^4.21.2" + } +} diff --git a/src/app.js b/src/app.js new file mode 100644 index 0000000..74d2846 --- /dev/null +++ b/src/app.js @@ -0,0 +1,181 @@ +import express from "express"; +import { httpProbe, probeResult, OK, ABSENT, UNOBSERVABLE } from "./probe.js"; +import { + UpstreamError, + embedTexts, + ensureCollection, + upsertPoints, + searchPoints, +} from "./upstreams.js"; + +/** + * Build the express app. `fetchImpl` is injectable so unit tests run without + * Qdrant/Ollama up. + */ +export function createApp(config, { fetchImpl = fetch } = {}) { + const app = express(); + app.use(express.json({ limit: "5mb" })); + + const upstreamOpts = { + ollamaUrl: config.ollamaUrl, + qdrantUrl: config.qdrantUrl, + embedModel: config.embedModel, + fetchImpl, + timeoutMs: config.upstreamTimeoutMs, + }; + + // ---- health: three-valued, per-dependency, with exact surfaces ---------- + app.get("/health", async (_req, res) => { + const qdrantSurface = `${config.qdrantUrl}/readyz`; + const ollamaSurface = `${config.ollamaUrl}/api/tags`; + + let embedModelResult = null; + const [qdrant, ollama] = await Promise.all([ + httpProbe(qdrantSurface, { fetchImpl, timeoutMs: config.probeTimeoutMs }), + httpProbe(ollamaSurface, { + fetchImpl, + timeoutMs: config.probeTimeoutMs, + classify: (_res, bodyText) => { + // While we can see the tag list, also classify the embed model: + // present -> ok, listed-but-missing -> absent (a true gap, distinct + // from "could not look"). + try { + const models = JSON.parse(bodyText)?.models ?? []; + const found = models.some( + (m) => m?.name === config.embedModel || m?.name?.split(":")[0] === config.embedModel + ); + embedModelResult = probeResult(found ? OK : ABSENT, ollamaSurface, { + model: config.embedModel, + }); + } catch { + embedModelResult = probeResult(UNOBSERVABLE, ollamaSurface, { + model: config.embedModel, + reason: "tag list unparseable", + }); + } + return null; // do not override the reachability verdict + }, + }), + ]); + if (!embedModelResult) { + // Ollama itself was unobservable, so the model's presence is too — + // never report it as absent (or fine) when we could not look. + embedModelResult = probeResult(UNOBSERVABLE, ollamaSurface, { + model: config.embedModel, + reason: "ollama unobservable, model presence not measurable", + }); + } + + const deps = { qdrant, ollama, embed_model: embedModelResult }; + const allOk = Object.values(deps).every((d) => d.status === OK); + // Overall is NEVER plain "ok" while any dependency is unobservable/absent. + const overall = allOk ? "ok" : "degraded"; + res.status(allOk ? 200 : 503).json({ + status: overall, + service: "shre-embed", + observed_at: new Date().toISOString(), + dependencies: deps, + }); + }); + + // ---- helpers ------------------------------------------------------------ + function upstreamErrorBody(err) { + return { + error: err.message, + // The caller must be able to tell "failed" from "answer is empty/zero": + status: UNOBSERVABLE, + measurement_surface: err.surface, + observed_at: new Date().toISOString(), + ...(err.httpStatus ? { upstream_http_status: err.httpStatus } : {}), + }; + } + + // ---- POST /v1/embed ----------------------------------------------------- + app.post("/v1/embed", async (req, res) => { + const { texts } = req.body ?? {}; + if (!Array.isArray(texts) || texts.length === 0 || !texts.every((t) => typeof t === "string")) { + return res.status(400).json({ error: "body must be {texts: string[]} with at least one text" }); + } + try { + const vectors = await embedTexts(texts, upstreamOpts); + res.json({ + vectors, + model: config.embedModel, + measurement_surface: `${config.ollamaUrl}/api/embeddings`, + observed_at: new Date().toISOString(), + }); + } catch (err) { + if (err instanceof UpstreamError) return res.status(502).json(upstreamErrorBody(err)); + throw err; + } + }); + + // ---- POST /v1/index ----------------------------------------------------- + app.post("/v1/index", async (req, res) => { + const { collection, items } = req.body ?? {}; + if (typeof collection !== "string" || !collection) { + return res.status(400).json({ error: "body must include a non-empty 'collection' string" }); + } + if ( + !Array.isArray(items) || + items.length === 0 || + !items.every((it) => it && it.id !== undefined && typeof it.text === "string") + ) { + return res.status(400).json({ error: "body must include items: [{id, text, payload?}] with at least one item" }); + } + try { + const vectors = await embedTexts(items.map((it) => it.text), upstreamOpts); + await ensureCollection(collection, vectors[0].length, upstreamOpts); + const points = items.map((it, i) => ({ + id: it.id, + vector: vectors[i], + payload: { ...(it.payload ?? {}), text: it.text }, + })); + const result = await upsertPoints(collection, points, upstreamOpts); + res.json({ + indexed: items.length, + collection, + qdrant: result?.result ?? result, + measurement_surface: `${config.qdrantUrl}/collections/${collection}/points`, + observed_at: new Date().toISOString(), + }); + } catch (err) { + if (err instanceof UpstreamError) return res.status(502).json(upstreamErrorBody(err)); + throw err; + } + }); + + // ---- POST /v1/search ---------------------------------------------------- + app.post("/v1/search", async (req, res) => { + const { collection, query, limit } = req.body ?? {}; + if (typeof collection !== "string" || !collection || typeof query !== "string" || !query) { + return res.status(400).json({ error: "body must include non-empty 'collection' and 'query' strings" }); + } + const lim = Number.isInteger(limit) && limit > 0 ? Math.min(limit, 100) : 10; + try { + const [vector] = await embedTexts([query], upstreamOpts); + const result = await searchPoints(collection, vector, lim, upstreamOpts); + res.json({ + results: result.map((r) => ({ id: r.id, score: r.score, payload: r.payload ?? null })), + collection, + limit: lim, + measurement_surface: `${config.qdrantUrl}/collections/${collection}/points/search`, + observed_at: new Date().toISOString(), + }); + } catch (err) { + if (err instanceof UpstreamError) return res.status(502).json(upstreamErrorBody(err)); + throw err; + } + }); + + // express error fallthrough (bad JSON etc.) + // eslint-disable-next-line no-unused-vars + app.use((err, _req, res, _next) => { + if (err?.type === "entity.parse.failed") { + return res.status(400).json({ error: "invalid JSON body" }); + } + res.status(500).json({ error: "internal error" }); + }); + + return app; +} diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..d84caca --- /dev/null +++ b/src/config.js @@ -0,0 +1,16 @@ +// Config via environment with estate defaults. +// +// NOTE: default OLLAMA_URL is 127.0.0.1:11436 (native Ollama), NOT 11434. +// On this Mac, 127.0.0.1:11434 is a known-broken Colima SSH-mux forward that +// accepts connections (tags respond) but hangs on generation/embedding. +export function loadConfig(env = process.env) { + return { + port: Number(env.PORT) > 0 ? Number(env.PORT) : 5499, + qdrantUrl: (env.QDRANT_URL || "http://127.0.0.1:6333").replace(/\/+$/, ""), + ollamaUrl: (env.OLLAMA_URL || "http://127.0.0.1:11436").replace(/\/+$/, ""), + embedModel: env.EMBED_MODEL || "nomic-embed-text", + // Per-request timeout for dependency probes and upstream calls (ms). + probeTimeoutMs: Number(env.PROBE_TIMEOUT_MS) > 0 ? Number(env.PROBE_TIMEOUT_MS) : 2500, + upstreamTimeoutMs: Number(env.UPSTREAM_TIMEOUT_MS) > 0 ? Number(env.UPSTREAM_TIMEOUT_MS) : 30000, + }; +} diff --git a/src/probe.js b/src/probe.js new file mode 100644 index 0000000..71bf2d9 --- /dev/null +++ b/src/probe.js @@ -0,0 +1,67 @@ +// Three-valued observability primitives. +// +// Every probe answer is one of THREE states, never two: +// "ok" — the probe saw the thing and it is present/working +// "absent" — the probe SAW the surface and the thing is genuinely missing +// "unobservable" — the probe COULD NOT SEE (dependency down, timeout, auth +// failure, endpoint missing). Never collapsed into ok/absent. +// +// Every result carries measurement_surface (exactly what was measured) and +// observed_at (ISO timestamp). + +export const OK = "ok"; +export const ABSENT = "absent"; +export const UNOBSERVABLE = "unobservable"; + +export function probeResult(status, measurementSurface, extra = {}) { + return { + status, + measurement_surface: measurementSurface, + observed_at: new Date().toISOString(), + ...extra, + }; +} + +/** + * Perform an HTTP GET probe of `url`. Returns a three-valued probeResult: + * - ok when the response status passes `okWhen` (default: 2xx) + * - unobservable on network error, timeout, or non-passing HTTP status + * (a 401/404/500 from the surface means we could not observe the truth, + * not that the dependency is healthy or absent). + * `classify(response, bodyText)` may optionally downgrade an HTTP-ok response + * to ABSENT (e.g. endpoint reachable but required model not installed). + */ +export async function httpProbe(url, { fetchImpl = fetch, timeoutMs = 2500, classify } = {}) { + const started = Date.now(); + let res; + try { + res = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (err) { + return probeResult(UNOBSERVABLE, url, { + reason: `fetch failed: ${err?.cause?.code || err?.name || "error"}: ${err?.message}`, + latency_ms: Date.now() - started, + }); + } + const latency = Date.now() - started; + if (!res.ok) { + return probeResult(UNOBSERVABLE, url, { + reason: `HTTP ${res.status} from probe surface`, + http_status: res.status, + latency_ms: latency, + }); + } + if (classify) { + let bodyText = ""; + try { + bodyText = await res.text(); + } catch { + return probeResult(UNOBSERVABLE, url, { + reason: "response body unreadable", + latency_ms: latency, + }); + } + const classified = classify(res, bodyText); + if (classified) return probeResult(classified.status, url, { latency_ms: latency, ...classified.extra }); + } + return probeResult(OK, url, { http_status: res.status, latency_ms: latency }); +} diff --git a/src/server.js b/src/server.js new file mode 100644 index 0000000..18f728f --- /dev/null +++ b/src/server.js @@ -0,0 +1,13 @@ +import { loadConfig } from "./config.js"; +import { createApp } from "./app.js"; + +const config = loadConfig(); +const app = createApp(config); + +app.listen(config.port, "127.0.0.1", () => { + // eslint-disable-next-line no-console + console.log( + `shre-embed listening on 127.0.0.1:${config.port} ` + + `(qdrant=${config.qdrantUrl} ollama=${config.ollamaUrl} model=${config.embedModel})` + ); +}); diff --git a/src/upstreams.js b/src/upstreams.js new file mode 100644 index 0000000..68e5f13 --- /dev/null +++ b/src/upstreams.js @@ -0,0 +1,126 @@ +// Thin clients for Ollama (embeddings) and Qdrant (vector store). +// All calls take an injected fetchImpl so tests run without either service up. +// +// Failures are surfaced as UpstreamError carrying the exact measurement +// surface (URL) that failed — callers propagate this so an API consumer can +// always tell "the answer is X" apart from "the answer was unobservable". + +export class UpstreamError extends Error { + constructor(message, { surface, cause, httpStatus } = {}) { + super(message); + this.name = "UpstreamError"; + this.surface = surface; + this.httpStatus = httpStatus; + if (cause) this.cause = cause; + } +} + +async function postJson(url, body, { fetchImpl, timeoutMs }) { + let res; + try { + res = await fetchImpl(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + throw new UpstreamError(`request failed: ${err?.message}`, { surface: url, cause: err }); + } + let json; + try { + json = await res.json(); + } catch { + json = undefined; + } + if (!res.ok) { + throw new UpstreamError(`HTTP ${res.status}: ${JSON.stringify(json)?.slice(0, 300)}`, { + surface: url, + httpStatus: res.status, + }); + } + return json; +} + +/** Embed each text via Ollama's embeddings API. Returns number[][]. */ +export async function embedTexts(texts, { ollamaUrl, embedModel, fetchImpl = fetch, timeoutMs = 30000 }) { + const url = `${ollamaUrl}/api/embeddings`; + const vectors = []; + for (const text of texts) { + const json = await postJson(url, { model: embedModel, prompt: text }, { fetchImpl, timeoutMs }); + if (!Array.isArray(json?.embedding)) { + throw new UpstreamError("Ollama response missing 'embedding' array", { surface: url }); + } + vectors.push(json.embedding); + } + return vectors; +} + +/** Ensure a Qdrant collection exists with the given vector size (Cosine). */ +export async function ensureCollection(collection, vectorSize, { qdrantUrl, fetchImpl = fetch, timeoutMs = 30000 }) { + const checkUrl = `${qdrantUrl}/collections/${encodeURIComponent(collection)}`; + let res; + try { + res = await fetchImpl(checkUrl, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (err) { + throw new UpstreamError(`request failed: ${err?.message}`, { surface: checkUrl, cause: err }); + } + if (res.ok) return { created: false }; + if (res.status !== 404) { + throw new UpstreamError(`HTTP ${res.status} checking collection`, { surface: checkUrl, httpStatus: res.status }); + } + let createRes; + try { + createRes = await fetchImpl(checkUrl, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ vectors: { size: vectorSize, distance: "Cosine" } }), + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + throw new UpstreamError(`request failed: ${err?.message}`, { surface: checkUrl, cause: err }); + } + if (!createRes.ok) { + throw new UpstreamError(`HTTP ${createRes.status} creating collection`, { surface: checkUrl, httpStatus: createRes.status }); + } + return { created: true }; +} + +/** Upsert points into a Qdrant collection. */ +export async function upsertPoints(collection, points, { qdrantUrl, fetchImpl = fetch, timeoutMs = 30000 }) { + const url = `${qdrantUrl}/collections/${encodeURIComponent(collection)}/points?wait=true`; + let res; + try { + res = await fetchImpl(url, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ points }), + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + throw new UpstreamError(`request failed: ${err?.message}`, { surface: url, cause: err }); + } + let json; + try { + json = await res.json(); + } catch { + json = undefined; + } + if (!res.ok) { + throw new UpstreamError(`HTTP ${res.status}: ${JSON.stringify(json)?.slice(0, 300)}`, { + surface: url, + httpStatus: res.status, + }); + } + return json; +} + +/** Vector search in a Qdrant collection. Returns scored points with payloads. */ +export async function searchPoints(collection, vector, limit, { qdrantUrl, fetchImpl = fetch, timeoutMs = 30000 }) { + const url = `${qdrantUrl}/collections/${encodeURIComponent(collection)}/points/search`; + const json = await postJson(url, { vector, limit, with_payload: true }, { fetchImpl, timeoutMs }); + if (!Array.isArray(json?.result)) { + throw new UpstreamError("Qdrant response missing 'result' array", { surface: url }); + } + return json.result; +} diff --git a/test/app.test.js b/test/app.test.js new file mode 100644 index 0000000..333da93 --- /dev/null +++ b/test/app.test.js @@ -0,0 +1,313 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { loadConfig } from "../src/config.js"; +import { createApp } from "../src/app.js"; + +// --------------------------------------------------------------------------- +// Helpers: run the app on an ephemeral loopback port with an injected fetch, +// so no Qdrant/Ollama needs to be up. +// --------------------------------------------------------------------------- + +const TEST_ENV = { + QDRANT_URL: "http://qdrant.test:6333", + OLLAMA_URL: "http://ollama.test:11436", + EMBED_MODEL: "nomic-embed-text", + PROBE_TIMEOUT_MS: "200", + UPSTREAM_TIMEOUT_MS: "500", +}; + +async function withServer(fetchImpl, fn) { + const config = loadConfig(TEST_ENV); + const app = createApp(config, { fetchImpl }); + const server = app.listen(0, "127.0.0.1"); + await once(server, "listening"); + const base = `http://127.0.0.1:${server.address().port}`; + try { + return await fn(base, config); + } finally { + server.close(); + } +} + +function jsonResponse(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +/** Fake fetch that dispatches on method+url via a routes table. */ +function fakeFetch(routes) { + const calls = []; + const impl = async (url, init = {}) => { + const method = init.method ?? "GET"; + const key = `${method} ${url}`; + calls.push({ key, body: init.body ? JSON.parse(init.body) : undefined }); + for (const [pattern, handler] of Object.entries(routes)) { + if (key === pattern || key.startsWith(pattern)) { + return handler({ url, init, body: init.body ? JSON.parse(init.body) : undefined }); + } + } + throw new TypeError(`fetch failed: no route for ${key}`); + }; + impl.calls = calls; + return impl; +} + +const DIM = 4; +const vecFor = (text) => [text.length, 1, 2, 3]; + +function happyRoutes() { + return { + // health probes + "GET http://qdrant.test:6333/readyz": () => new Response("all shards ready", { status: 200 }), + "GET http://ollama.test:11436/api/tags": () => + jsonResponse({ models: [{ name: "nomic-embed-text:latest" }, { name: "qwen3:30b-a3b" }] }), + // embeddings + "POST http://ollama.test:11436/api/embeddings": ({ body }) => jsonResponse({ embedding: vecFor(body.prompt) }), + // qdrant collection lifecycle + "GET http://qdrant.test:6333/collections/notes": () => jsonResponse({ result: {} }, 404), + "PUT http://qdrant.test:6333/collections/notes/points": () => + jsonResponse({ result: { operation_id: 1, status: "completed" } }), + "PUT http://qdrant.test:6333/collections/notes": () => jsonResponse({ result: true }), + "POST http://qdrant.test:6333/collections/notes/points/search": () => + jsonResponse({ + result: [ + { id: "a", score: 0.91, payload: { text: "alpha", src: "t" } }, + { id: "b", score: 0.42, payload: { text: "beta" } }, + ], + }), + }; +} + +// --------------------------------------------------------------------------- +// config +// --------------------------------------------------------------------------- + +test("config defaults: port 5499, qdrant 6333, ollama 11436 (NOT 11434), nomic-embed-text", () => { + const c = loadConfig({}); + assert.equal(c.port, 5499); + assert.equal(c.qdrantUrl, "http://127.0.0.1:6333"); + assert.equal(c.ollamaUrl, "http://127.0.0.1:11436"); + assert.ok(!c.ollamaUrl.includes("11434"), "must never default to the broken 11434 forward"); + assert.equal(c.embedModel, "nomic-embed-text"); +}); + +// --------------------------------------------------------------------------- +// /health — three-valued +// --------------------------------------------------------------------------- + +test("/health reports ok with surfaces + observed_at when all deps respond", async () => { + await withServer(fakeFetch(happyRoutes()), async (base) => { + const res = await fetch(`${base}/health`); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.status, "ok"); + assert.equal(body.dependencies.qdrant.status, "ok"); + assert.equal(body.dependencies.qdrant.measurement_surface, "http://qdrant.test:6333/readyz"); + assert.equal(body.dependencies.ollama.status, "ok"); + assert.equal(body.dependencies.ollama.measurement_surface, "http://ollama.test:11436/api/tags"); + assert.equal(body.dependencies.embed_model.status, "ok"); + for (const dep of Object.values(body.dependencies)) { + assert.ok(Date.parse(dep.observed_at), "every dep carries observed_at"); + } + }); +}); + +test("/health reports unobservable (never healthy) when deps are down", async () => { + // fetch that always fails = both dependencies down + const deadFetch = async () => { + throw new TypeError("fetch failed: ECONNREFUSED"); + }; + await withServer(deadFetch, async (base) => { + const res = await fetch(`${base}/health`); + assert.equal(res.status, 503); + const body = await res.json(); + assert.equal(body.status, "degraded"); + assert.notEqual(body.status, "ok", "must never report plain healthy while a dep is unobservable"); + assert.equal(body.dependencies.qdrant.status, "unobservable"); + assert.match(body.dependencies.qdrant.reason, /fetch failed/); + assert.equal(body.dependencies.ollama.status, "unobservable"); + // model presence is ALSO unobservable — not "absent" — when ollama is down + assert.equal(body.dependencies.embed_model.status, "unobservable"); + assert.equal(body.dependencies.qdrant.measurement_surface, "http://qdrant.test:6333/readyz"); + }); +}); + +test("/health: dep returning HTTP 500 is unobservable, not ok and not absent", async () => { + const routes = happyRoutes(); + routes["GET http://qdrant.test:6333/readyz"] = () => new Response("boom", { status: 500 }); + await withServer(fakeFetch(routes), async (base) => { + const res = await fetch(`${base}/health`); + assert.equal(res.status, 503); + const body = await res.json(); + assert.equal(body.status, "degraded"); + assert.equal(body.dependencies.qdrant.status, "unobservable"); + assert.equal(body.dependencies.qdrant.http_status, 500); + assert.equal(body.dependencies.ollama.status, "ok"); + }); +}); + +test("/health: embed model missing from a REACHABLE ollama is absent (a true gap)", async () => { + const routes = happyRoutes(); + routes["GET http://ollama.test:11436/api/tags"] = () => jsonResponse({ models: [{ name: "qwen3:30b-a3b" }] }); + await withServer(fakeFetch(routes), async (base) => { + const res = await fetch(`${base}/health`); + const body = await res.json(); + assert.equal(body.dependencies.ollama.status, "ok", "ollama itself was observable"); + assert.equal(body.dependencies.embed_model.status, "absent"); + assert.equal(body.status, "degraded"); + assert.equal(res.status, 503); + }); +}); + +// --------------------------------------------------------------------------- +// /v1/embed +// --------------------------------------------------------------------------- + +test("POST /v1/embed returns one vector per text via ollama", async () => { + const f = fakeFetch(happyRoutes()); + await withServer(f, async (base) => { + const res = await fetch(`${base}/v1/embed`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ texts: ["hello", "worlds!"] }), + }); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.vectors.length, 2); + assert.deepEqual(body.vectors[0], vecFor("hello")); + assert.deepEqual(body.vectors[1], vecFor("worlds!")); + assert.equal(body.measurement_surface, "http://ollama.test:11436/api/embeddings"); + assert.ok(Date.parse(body.observed_at)); + const embedCalls = f.calls.filter((c) => c.key.includes("/api/embeddings")); + assert.equal(embedCalls.length, 2); + assert.equal(embedCalls[0].body.model, "nomic-embed-text"); + }); +}); + +test("POST /v1/embed with ollama down returns 502 marked unobservable with surface", async () => { + const deadFetch = async () => { + throw new TypeError("fetch failed: ECONNREFUSED"); + }; + await withServer(deadFetch, async (base) => { + const res = await fetch(`${base}/v1/embed`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ texts: ["hello"] }), + }); + assert.equal(res.status, 502); + const body = await res.json(); + assert.equal(body.status, "unobservable"); + assert.equal(body.measurement_surface, "http://ollama.test:11436/api/embeddings"); + assert.ok(Date.parse(body.observed_at)); + }); +}); + +test("POST /v1/embed validates body", async () => { + await withServer(fakeFetch(happyRoutes()), async (base) => { + for (const bad of [{}, { texts: [] }, { texts: "hello" }, { texts: [1, 2] }]) { + const res = await fetch(`${base}/v1/embed`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(bad), + }); + assert.equal(res.status, 400); + } + }); +}); + +// --------------------------------------------------------------------------- +// /v1/index +// --------------------------------------------------------------------------- + +test("POST /v1/index embeds, auto-creates missing collection, upserts points", async () => { + const f = fakeFetch(happyRoutes()); + await withServer(f, async (base) => { + const res = await fetch(`${base}/v1/index`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + collection: "notes", + items: [ + { id: 1, text: "alpha", payload: { src: "t" } }, + { id: 2, text: "beta" }, + ], + }), + }); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.indexed, 2); + assert.equal(body.collection, "notes"); + // collection was created with the right vector size + const create = f.calls.find((c) => c.key === "PUT http://qdrant.test:6333/collections/notes"); + assert.ok(create, "collection create issued"); + assert.equal(create.body.vectors.size, DIM); + // points carried vectors and payload (text folded in) + const upsert = f.calls.find((c) => c.key.startsWith("PUT http://qdrant.test:6333/collections/notes/points")); + assert.equal(upsert.body.points.length, 2); + assert.deepEqual(upsert.body.points[0].vector, vecFor("alpha")); + assert.deepEqual(upsert.body.points[0].payload, { src: "t", text: "alpha" }); + }); +}); + +test("POST /v1/index with qdrant down returns 502 unobservable (never silently indexed:0)", async () => { + const routes = happyRoutes(); + const qdrantDead = ({ url }) => { + throw new TypeError("fetch failed: ECONNREFUSED " + url); + }; + routes["GET http://qdrant.test:6333/collections/notes"] = qdrantDead; + await withServer(fakeFetch(routes), async (base) => { + const res = await fetch(`${base}/v1/index`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ collection: "notes", items: [{ id: 1, text: "alpha" }] }), + }); + assert.equal(res.status, 502); + const body = await res.json(); + assert.equal(body.status, "unobservable"); + assert.equal(body.measurement_surface, "http://qdrant.test:6333/collections/notes"); + }); +}); + +// --------------------------------------------------------------------------- +// /v1/search +// --------------------------------------------------------------------------- + +test("POST /v1/search embeds the query and returns scored payloads", async () => { + const f = fakeFetch(happyRoutes()); + await withServer(f, async (base) => { + const res = await fetch(`${base}/v1/search`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ collection: "notes", query: "find alpha", limit: 2 }), + }); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.results.length, 2); + assert.deepEqual(body.results[0], { id: "a", score: 0.91, payload: { text: "alpha", src: "t" } }); + const search = f.calls.find((c) => c.key.startsWith("POST http://qdrant.test:6333/collections/notes/points/search")); + assert.deepEqual(search.body.vector, vecFor("find alpha")); + assert.equal(search.body.limit, 2); + assert.equal(search.body.with_payload, true); + }); +}); + +test("POST /v1/search with qdrant down returns 502 unobservable — never an empty result set", async () => { + const routes = happyRoutes(); + routes["POST http://qdrant.test:6333/collections/notes/points/search"] = () => { + throw new TypeError("fetch failed: ECONNREFUSED"); + }; + await withServer(fakeFetch(routes), async (base) => { + const res = await fetch(`${base}/v1/search`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ collection: "notes", query: "anything" }), + }); + assert.equal(res.status, 502); + const body = await res.json(); + assert.equal(body.status, "unobservable"); + assert.ok(!("results" in body), "an unobservable search must not look like an empty result"); + }); +});