diff --git a/.gitignore b/.gitignore index 800f083..7f1bfb0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *node_modules/ app/dist/ app/release/ +app/native/ frontend/dist/ +lib/target/ diff --git a/README.md b/README.md index 591f69d..28e75d3 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Beyond chat, Modelforge includes an **agentic mode** — the model can read/writ - [Quick start](#quick-start-try-it-in-5-minutes) - [Agent mode](#agent-mode) - [Building from source](#building-from-source) -- [Project structure](#project-structure) +- [Documentation](#documentation) - [Testing](#testing) - [Security](#security) - [Contributing](#contributing) @@ -140,31 +140,15 @@ If steps 2–3 work, the core app is functioning correctly — everything else l ## Agent mode -Click **Agent** in the chat toolbar and pick a folder — that becomes the model's sandboxed workspace for the rest of the conversation. The model can then call: - -| Tool | What it does | -|---|---| -| `read_file` | Read a text file, optionally by line range | -| `write_file` | Create or overwrite a file (creates parent directories as needed) | -| `replace_in_file` | Replace an exact text block without rewriting the whole file; supports Undo | -| `find_files`, `file_info` | Discover files by glob and inspect path metadata | -| `list_dir` | List files and subdirectories | -| `search_files` | Search for a text string across the workspace | -| `make_directory`, `move_path`, `delete_path` | Organize workspace files with explicit approval for mutations | -| `run_command` | Execute a shell command in the workspace (or a subfolder), with a 60s timeout | -| `run_code` | Run a Python or JavaScript snippet — a convenience over shell-quoting multi-line code through `run_command`, not a new capability | -| `git_status`, `git_diff`, `git_log` | Read-only git helpers (auto-approvable, like the file tools) so the model doesn't need to guess flag syntax | -| `git_commit` | Stage everything and commit — requires approval, like `write_file` | -| `github_list_repositories` | List repositories accessible to the linked GitHub account | -| `github_repository_tree` | Inspect a repository's complete file structure before analysis | -| `github_read_file` | Read selected files from public or private linked-account repositories | +Click **Agent** in the chat toolbar and pick a folder — that becomes the model's sandboxed workspace for the rest of the conversation. The model can then call a wide catalog of tools: file read/write/search/patch, shell and background commands, an interactive terminal, read-only and commit git operations, and network tools (web search, arbitrary HTTP requests, linked-GitHub repo access). **See [docs/AGENT_MODE.md](docs/AGENT_MODE.md) for the full, current tool table** — the summary below covers the safety model, which is the part worth understanding before you use it. **Safety model:** - Built-in filesystem tools are genuinely confined to the chosen workspace folder — path traversal (`../../etc`), absolute paths elsewhere on disk, and symlinks that resolve outside the workspace are rejected before anything runs. -- `run_command` and `run_code` are different: a shell command (or a script `run_code` hands to `python3`/`node`) is opaque text that can reference any path on the system regardless of its working directory, so neither is sandboxed the way the file tools are. As a safety net, commands (and `run_code`'s source text) matching destructive or system-level patterns — deleting outside the workspace, formatting a drive, shutting down the machine, registry deletion, `sudo`/`runas`, piping a remote script into a shell — are **rejected outright**, even if already approved. This blocklist catches the common catastrophic cases, not everything a shell or script can do — only approve a command or snippet you actually understand. -- Every call (including ones the blocklist doesn't catch) shows an **Allow / Deny** card before it executes — nothing runs without an explicit click. Read-only tools (`read_file`, `find_files`, `file_info`, `list_dir`, `search_files`, `git_status`, `git_diff`, `git_log`) can be marked "always allow this session" to cut down on repetitive approvals; filesystem mutations, `run_command`, `run_code`, and `git_commit` always require a fresh click, since they have real, potentially irreversible effects. +- `run_command` and `run_code` are different: a shell command (or a script `run_code` hands to `python3`/`node`) is opaque text that can reference any path on the system regardless of its working directory, so neither is confined the way the file tools are. Where the OS supports it, commands run inside a real sandbox instead (bubblewrap on Linux, `sandbox-exec` on macOS — Windows has no equivalent). As a safety net everywhere, commands (and `run_code`'s source text) matching destructive or system-level patterns — deleting outside the workspace, formatting a drive, shutting down the machine, registry deletion, `sudo`/`runas`, piping a remote script into a shell — are **rejected outright**, even if already approved. This blocklist catches the common catastrophic cases, not everything a shell or script can do — only approve a command or snippet you actually understand. +- Every call (including ones the blocklist doesn't catch) shows an **Allow / Deny** card before it executes — nothing runs without an explicit click. A narrow set of strictly read-only, no-network tools can be marked "always allow this session" to cut down on repetitive approvals; anything that mutates the workspace, runs code, or touches the network always requires a fresh click, since those have real, potentially irreversible or unattended-outbound-channel effects — see [docs/AGENT_MODE.md](docs/AGENT_MODE.md#safety-model) for exactly which tools qualify and why. - A per-turn step limit (25 tool-result → model-continuation round trips) stops a model from looping indefinitely without producing a final answer. - The trust list for "always allow" is in-memory only — closing and reopening a chat resets it. +- A Settings toggle can disable every network-capable tool at once, for a fully offline, filesystem-and-shell-only workflow. **Preview & Rollback:** - A pending `write_file` call shows a real **line-by-line diff** against the file's current content (or a "new file" badge if it doesn't exist yet) instead of a raw argument dump, so you can see exactly what would change before clicking Allow. @@ -178,7 +162,7 @@ Click **Agent** in the chat toolbar and pick a folder — that becomes the model Enabled servers reconnect automatically on launch; each server's tools appear in Agent mode's tool list prefixed with the server's name, going through the exact same Allow/Deny approval flow as built-in tools. (SSE and plain WebSocket transports aren't implemented — SSE is the legacy MCP HTTP transport, now superseded by Streamable HTTP, and WebSocket isn't part of the MCP spec itself.) -**Model choice matters.** Agent mode works with whatever model you point it at, but only actually produces tool calls if that model was trained for function/tool calling — a model without that training will just chat normally and never call a tool. The Settings model browser flags models with reliable tool-calling support with a 🔧 **Tool calling** badge (e.g. the Qwen3 family, Llama 3.1+, Mistral Nemo, Qwen2.5-Coder, Devstral). +**Model choice matters.** Agent mode works with whatever model you point it at, but only actually produces tool calls if that model was trained for function/tool calling — a model without that training will just chat normally and never call a tool. The Settings model browser flags models with reliable tool-calling support with a 🔧 **Tool calling** badge (e.g. the Qwen3 family, Llama 3.1+, Mistral Nemo, Qwen2.5-Coder, Devstral). The llama.cpp backend doesn't have tool-calling wired up yet — see [Features](#features) above. ## Building from source @@ -199,26 +183,18 @@ npm run dev --prefix app npm run package --prefix app ``` -Packaged installers are written to `app/release/`. +Packaged installers are written to `app/release/`. For the full development workflow (hot reload, +rebuilding the native Rust addon, cross-platform packaging details, CI/release pipeline), see +[docs/DEVELOPMENT.md](docs/DEVELOPMENT.md). -## Project structure +## Documentation -``` -frontend/ React + Vite renderer (the UI) - src/pages/ Chat and Settings screens - src/components/ Shared UI (layout, command palette, markdown rendering, shadcn primitives) - src/lib/ i18n, model catalogs, pricing estimates, provider helpers - -app/ Electron main process - src/main.ts Window management, IPC handler registration - src/providers/ Ollama/OpenAI/Anthropic chat + tool-calling adapters - src/agent-tools.ts Agent mode's file/shell tool implementations (workspace-sandboxed) - src/*-store.ts Settings/sessions/projects/secrets persistence (atomic writes, corruption recovery) - src/rag.ts Chunking + embedding + retrieval for large folder attachments - src/logger.ts Rotating file logs surfaced via Settings → Data → Diagnostics -``` +This README covers what the app does and how to install/build it. For anything deeper: -The frontend builds to a single inlined HTML file (`vite-plugin-singlefile`) so Electron can load it directly via `file://` in production, matching how the packaged app actually runs. +- **[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)** — how the Electron main/renderer/preload split works, the IPC bridge, the native Rust downloader addon, the persistence pattern (atomic writes, corruption recovery), and the end-to-end data flow for sending a chat message. +- **[docs/AGENT_MODE.md](docs/AGENT_MODE.md)** — the full Agent mode tool catalog, the workspace-sandboxing and OS-level command-sandboxing model, the tool-approval policy and exactly why each tool is or isn't auto-approvable, and MCP server integration. +- **[docs/DEVELOPMENT.md](docs/DEVELOPMENT.md)** — prerequisites, running in dev mode, testing, building, packaging installers for each platform, and the project's directory layout. +- **[ml/hardware-recommender/README.md](ml/hardware-recommender/README.md)** — the standalone Python project that trains the hardware/model-fit recommender. Its training pipeline isn't part of the app's build, but its trained checkpoint ships with the app and is used at runtime via a Python worker to enhance the Models & hardware recommendations mentioned above. ## Testing @@ -227,19 +203,19 @@ npm test --prefix frontend npm test --prefix app ``` -The `app` suite covers the store layer (atomic writes, corrupted-file recovery), the agent tools (including path-traversal rejection and shell command execution), and the RAG chunking/similarity logic. Both suites run in CI on every push and pull request via [`.github/workflows/ci.yml`](.github/workflows/ci.yml), which also lints, typechecks, and builds both packages. +The `app` suite covers the store layer (atomic writes, corrupted-file recovery), the agent tools (including path-traversal rejection and shell command execution), and the RAG chunking/similarity logic. Both suites run in CI on every push and pull request via [`.github/workflows/ci.yml`](.github/workflows/ci.yml), which also lints, typechecks, and builds both packages. See [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md#testing) for what each suite specifically exercises. ## Security - **Process isolation**: `contextIsolation: true`, `nodeIntegration: false` — the renderer only ever talks to the main process through an explicit, typed preload bridge. - **Content Security Policy** restricting plugins, frames, and form submissions; external links open in your default browser instead of an unmanaged Electron window. -- **API keys** are encrypted at rest via the OS credential store (`safeStorage`) and never leave the device. -- **Agent mode** tool calls are workspace-sandboxed (path-traversal rejected) and require explicit per-call approval — see [Agent mode](#agent-mode) above. +- **API keys** are encrypted at rest via the OS credential store (`safeStorage`) and never leave the device. On the rare system with no OS credential store available (e.g. some keyring-less Linux setups), keys fall back to being stored in plain text on disk rather than being silently dropped — Settings shows a prominent warning in that case rather than the normal "encrypted" note. +- **Agent mode** tool calls are workspace-sandboxed (path-traversal rejected) and require explicit per-call approval — see [Agent mode](#agent-mode) above and [docs/AGENT_MODE.md](docs/AGENT_MODE.md) for the full detail. - No telemetry, no analytics, no data sent anywhere except directly to whichever provider (Ollama, OpenAI, Anthropic) you've configured. ## Contributing -Issues and pull requests are welcome. Before opening a PR, please make sure: +Issues and pull requests are welcome. See [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) for the full setup and testing workflow. Before opening a PR, please make sure: ```sh npm run lint --prefix frontend diff --git a/app/package-lock.json b/app/package-lock.json index f4481fa..f0a7fc3 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,23 +1,28 @@ { "name": "app", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "app", - "version": "1.0.0", + "version": "1.1.0", "license": "MIT", "dependencies": { + "better-sqlite3": "^13.0.1", "electron-updater": "^6.6.2", "ffmpeg-static": "^5.3.0", "node-llama-cpp": "^3.19.1", + "node-pty": "^1.1.0", "pdf-parse": "^2.4.5", + "pidusage": "^4.0.1", "tesseract.js": "^5.1.1" }, "devDependencies": { + "@types/better-sqlite3": "^7.6.13", "@types/node": "^26.1.1", "@types/pdf-parse": "^1.1.5", + "@types/pidusage": "^2.0.5", "concurrently": "^10.0.3", "cross-env": "^10.1.0", "electron": "^43.1.1", @@ -1990,6 +1995,16 @@ "url": "https://github.com/tinyhttp/tinyhttp?sponsor=1" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -2092,6 +2107,13 @@ "@types/node": "*" } }, + "node_modules/@types/pidusage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/pidusage/-/pidusage-2.0.5.tgz", + "integrity": "sha512-MIiyZI4/MK9UGUXWt0jJcCZhVw7YdhBuTOuqP/BjuLDLZ2PmmViMIQgZiWxtaMicQfAz/kMrZ5T7PKxFSkTeUA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/responselike": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz", @@ -2650,6 +2672,19 @@ ], "license": "MIT" }, + "node_modules/better-sqlite3": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.1.tgz", + "integrity": "sha512-LYpmOXdkpQYf4wmlxkdzW01XGlOXNIbjLg45yNkh0FQ4814VbK9PdOFmhZpYbej+EZtR/i3FDdhEG98HqZdgnA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.0.0" + }, + "engines": { + "node": ">=22" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -5799,6 +5834,22 @@ "node": ">=12" } }, + "node_modules/node-pty": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/node-pty/-/node-pty-1.1.0.tgz", + "integrity": "sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^7.1.0" + } + }, + "node_modules/node-pty/node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, "node_modules/nopt": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-9.0.0.tgz", @@ -6069,6 +6120,38 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pidusage": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pidusage/-/pidusage-4.0.1.tgz", + "integrity": "sha512-yCH2dtLHfEBnzlHUJymR/Z1nN2ePG3m392Mv8TFlTP1B0xkpMQNHAnfkY0n2tAi6ceKO6YWhxYfZ96V4vVkh/g==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/pidusage/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/pkijs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.0.tgz", diff --git a/app/package.json b/app/package.json index 463b4f8..889dc64 100644 --- a/app/package.json +++ b/app/package.json @@ -1,16 +1,18 @@ { "name": "app", - "version": "1.0.0", + "version": "1.1.0", "description": "", "main": "dist/main.js", "scripts": { + "postinstall": "node scripts/fix-native-permissions.js", "test": "vitest run", "build": "tsc -p tsconfig.json", "dev:renderer": "npm --prefix ../frontend run dev", "dev:main": "tsc -p tsconfig.json --watch --preserveWatchOutput", "dev:electron": "wait-on tcp:5173 dist/main.js && cross-env NODE_ENV=development electron .", "dev": "concurrently -k -s first \"npm:dev:renderer\" \"npm:dev:main\" \"npm:dev:electron\"", - "build:all": "npm --prefix ../frontend run build && npm run build", + "build:native": "npm --prefix ../lib install && npm --prefix ../lib run build", + "build:all": "npm run build:native && npm --prefix ../frontend run build && npm run build", "package": "npm run build:all && electron-builder" }, "keywords": [], @@ -25,6 +27,7 @@ }, "files": [ "dist/**/*", + "native/**/*", "node_modules/**/*", "package.json" ], @@ -33,12 +36,19 @@ "**/node_modules/tesseract.js/**", "**/node_modules/tesseract.js-core/**", "**/node_modules/node-llama-cpp/**", - "**/node_modules/@node-llama-cpp/**" + "**/node_modules/@node-llama-cpp/**", + "**/node_modules/node-pty/**", + "**/node_modules/better-sqlite3/**", + "native/**" ], "extraResources": [ { "from": "../frontend/dist", "to": "frontend-dist" + }, + { + "from": "python", + "to": "python" } ], "win": { @@ -76,8 +86,10 @@ } }, "devDependencies": { + "@types/better-sqlite3": "^7.6.13", "@types/node": "^26.1.1", "@types/pdf-parse": "^1.1.5", + "@types/pidusage": "^2.0.5", "concurrently": "^10.0.3", "cross-env": "^10.1.0", "electron": "^43.1.1", @@ -87,10 +99,13 @@ "wait-on": "^9.0.10" }, "dependencies": { + "better-sqlite3": "^13.0.1", "electron-updater": "^6.6.2", "ffmpeg-static": "^5.3.0", "node-llama-cpp": "^3.19.1", + "node-pty": "^1.1.0", "pdf-parse": "^2.4.5", + "pidusage": "^4.0.1", "tesseract.js": "^5.1.1" } } diff --git a/app/python/artifacts/hardware_recommender.pt b/app/python/artifacts/hardware_recommender.pt new file mode 100644 index 0000000..1667eb2 Binary files /dev/null and b/app/python/artifacts/hardware_recommender.pt differ diff --git a/app/python/recommender_worker.py b/app/python/recommender_worker.py new file mode 100644 index 0000000..ceb7405 --- /dev/null +++ b/app/python/recommender_worker.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# Long-lived JSON-line worker (same protocol as runtime_worker.py) serving +# predictions from the trained hardware-recommender model +# (ml/hardware-recommender/, shipped as artifacts/hardware_recommender.pt +# alongside this script). Model/feature-encoding logic here is a trimmed, +# inference-only copy of ml/hardware-recommender/recommender/model.py — kept +# separate (rather than imported) so the packaged app/python/ directory +# stays self-contained and doesn't need the training project's pandas/pyarrow +# dependencies just to run inference. +import json +import os +import sys +import time +from pathlib import Path + +import torch +from torch import nn + +PROTOCOL_VERSION = 1 +STARTED = time.monotonic() + +PLATFORMS = ["windows", "linux", "macos"] +GPU_BACKENDS = ["none", "cuda", "rocm", "metal", "vulkan", "directml"] +FIT_CLASSES = ["Runs comfortably", "May require partial GPU offload", "CPU only", "Insufficient memory"] +RUNTIME_CLASSES = ["llama.cpp-cpu", "llama.cpp-cuda", "llama.cpp-rocm", "llama.cpp-metal", "llama.cpp-vulkan", "mlx", "vllm"] +CONTEXT_CLASSES = [2048, 4096, 8192, 16384, 32768, 65536, 131072] +NUMERIC_FEATURES = ["model_params_b", "quality_score", "is_moe", "quant_bits", "ram_gb", "vram_gb", "cpu_cores"] +QUANTIZATIONS = {"Q2_K": 2.625, "Q3_K_M": 3.50, "Q4_K_M": 4.75, "Q5_K_M": 5.50, "Q6_K": 6.56, "Q8_0": 8.50} +STATUS_ORDER = {"Runs comfortably": 3, "May require partial GPU offload": 2, "CPU only": 1, "Insufficient memory": 0} + + +class HardwareRecommender(nn.Module): + def __init__(self, hidden_size=64): + super().__init__() + input_size = len(NUMERIC_FEATURES) + len(PLATFORMS) + len(GPU_BACKENDS) + # Dropout layers are no-ops in eval() but their presence still shifts + # nn.Sequential's positional indices — they have to stay in these + # exact slots to match the trained checkpoint's state_dict keys + # (see ml/hardware-recommender/recommender/model.py, the source of + # truth this architecture is copied from). + self.backbone = nn.Sequential( + nn.Linear(input_size, hidden_size), + nn.LayerNorm(hidden_size), + nn.SiLU(), + nn.Dropout(0.10), + nn.Linear(hidden_size, hidden_size), + nn.SiLU(), + nn.Dropout(0.10), + nn.Linear(hidden_size, hidden_size // 2), + nn.SiLU(), + ) + shared = hidden_size // 2 + self.fit_head = nn.Linear(shared, len(FIT_CLASSES)) + self.runtime_head = nn.Linear(shared, len(RUNTIME_CLASSES)) + self.context_head = nn.Linear(shared, len(CONTEXT_CLASSES)) + self.speed_head = nn.Sequential(nn.Linear(shared, 1), nn.Softplus()) + + def forward(self, features): + hidden = self.backbone(features) + return { + "fit": self.fit_head(hidden), + "runtime": self.runtime_head(hidden), + "context": self.context_head(hidden), + "log_speed": self.speed_head(hidden).squeeze(-1), + } + + +def log(level, event, **fields): + print(json.dumps({"timestamp": time.time(), "level": level, "event": event, **fields}), file=sys.stderr, flush=True) + + +def response(request_id, ok, result=None, code=None, message=None): + payload = {"protocol": PROTOCOL_VERSION, "id": request_id, "ok": ok} + if ok: + payload["result"] = result + else: + payload["error"] = {"code": code, "message": message} + print(json.dumps(payload), flush=True) + + +_model = None +_normalizer_mean = None +_normalizer_std = None + + +def load(): + global _model, _normalizer_mean, _normalizer_std + if _model is not None: + return + checkpoint_path = Path(__file__).parent / "artifacts" / "hardware_recommender.pt" + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + model = HardwareRecommender(hidden_size=int(checkpoint["hidden_size"])) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + _model = model + _normalizer_mean = checkpoint["normalizer"]["mean"] + _normalizer_std = checkpoint["normalizer"]["std"] + + +def encode_row(hardware, quant_bits): + numeric = [ + float(hardware["model_params_b"]), + float(hardware.get("quality_score", 30.0)), + 1.0 if hardware.get("is_moe") else 0.0, + float(quant_bits), + float(hardware["ram_gb"]), + float(hardware["vram_gb"]), + float(hardware["cpu_cores"]), + ] + numeric = [(value - mean) / std for value, mean, std in zip(numeric, _normalizer_mean, _normalizer_std)] + platform_onehot = [1.0 if hardware["platform"] == name else 0.0 for name in PLATFORMS] + backend_onehot = [1.0 if hardware["gpu_backend"] == name else 0.0 for name in GPU_BACKENDS] + return numeric + platform_onehot + backend_onehot + + +def recommend(hardware): + load() + rows = [encode_row(hardware, bits) for bits in QUANTIZATIONS.values()] + features = torch.tensor(rows, dtype=torch.float32) + with torch.inference_mode(): + output = _model(features) + fit_indices = output["fit"].argmax(1).tolist() + runtime_indices = output["runtime"].argmax(1).tolist() + context_indices = output["context"].argmax(1).tolist() + speeds = torch.expm1(output["log_speed"]).clamp_min(0).tolist() + + candidates = [] + for index, quantization in enumerate(QUANTIZATIONS): + candidates.append({ + "quantization": quantization, + "fit": FIT_CLASSES[fit_indices[index]], + "runtime": RUNTIME_CLASSES[runtime_indices[index]], + "estimatedContextTokens": CONTEXT_CLASSES[context_indices[index]], + "estimatedTokensPerSecond": round(float(speeds[index]), 1), + }) + + comfortable = [c for c in candidates if c["fit"] == "Runs comfortably"] + viable = [c for c in candidates if STATUS_ORDER[c["fit"]] > 0] + if comfortable: + choice = comfortable[-1] + elif viable: + choice = next((c for c in viable if c["quantization"] == "Q4_K_M"), viable[0]) + else: + choice = candidates[0] + selected = dict(choice) + selected["allQuantizations"] = candidates + return selected + + +log("info", "worker_started", protocol=PROTOCOL_VERSION, pid=os.getpid()) +for line in sys.stdin: + request_id = "" + try: + request = json.loads(line) + request_id = str(request.get("id", "")) + if request.get("protocol") != PROTOCOL_VERSION: + response(request_id, False, code="protocol_mismatch", message=f"Expected protocol {PROTOCOL_VERSION}") + continue + method = request.get("method") + if method == "health": + response(request_id, True, {"status": "ok", "protocol": PROTOCOL_VERSION, "pid": os.getpid()}) + elif method == "recommend": + response(request_id, True, recommend(request.get("params", {}))) + elif method == "shutdown": + response(request_id, True, {"shuttingDown": True}) + log("info", "worker_shutdown") + break + else: + response(request_id, False, code="method_not_found", message=f"Unknown method: {method}") + except Exception as error: + log("error", "request_failed", error=str(error)) + response(request_id, False, code="invalid_request", message=str(error)) diff --git a/app/python/runtime_worker.py b/app/python/runtime_worker.py new file mode 100644 index 0000000..e4f0350 --- /dev/null +++ b/app/python/runtime_worker.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json +import os +import platform +import resource +import sys +import time + +PROTOCOL_VERSION = 1 +STARTED = time.monotonic() + +def log(level, event, **fields): + print(json.dumps({"timestamp": time.time(), "level": level, "event": event, **fields}), file=sys.stderr, flush=True) + +def response(request_id, ok, result=None, code=None, message=None): + payload = {"protocol": PROTOCOL_VERSION, "id": request_id, "ok": ok} + if ok: payload["result"] = result + else: payload["error"] = {"code": code, "message": message} + print(json.dumps(payload), flush=True) + +log("info", "worker_started", protocol=PROTOCOL_VERSION, pid=os.getpid()) +for line in sys.stdin: + try: + request = json.loads(line) + request_id = str(request.get("id", "")) + if request.get("protocol") != PROTOCOL_VERSION: + response(request_id, False, code="protocol_mismatch", message=f"Expected protocol {PROTOCOL_VERSION}") + continue + method = request.get("method") + if method == "health": + response(request_id, True, {"status": "ok", "protocol": PROTOCOL_VERSION, "pid": os.getpid(), "python": platform.python_version(), "uptimeSeconds": time.monotonic() - STARTED}) + elif method == "metrics": + usage = resource.getrusage(resource.RUSAGE_SELF) + response(request_id, True, {"uptimeSeconds": time.monotonic() - STARTED, "maxRssKb": usage.ru_maxrss, "userCpuSeconds": usage.ru_utime, "systemCpuSeconds": usage.ru_stime}) + elif method == "shutdown": + response(request_id, True, {"shuttingDown": True}); log("info", "worker_shutdown"); break + else: + response(request_id, False, code="method_not_found", message=f"Unknown method: {method}") + except Exception as error: + log("error", "request_failed", error=str(error)) + response("", False, code="invalid_request", message=str(error)) diff --git a/app/scripts/fix-native-permissions.js b/app/scripts/fix-native-permissions.js new file mode 100644 index 0000000..95aa2d1 --- /dev/null +++ b/app/scripts/fix-native-permissions.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// node-pty ships a `spawn-helper` executable in its darwin/linux prebuilds — +// used to fork+exec pty child processes (macOS specifically needs a separate +// exec'd helper rather than a plain fork, since forking a process that has +// loaded Objective-C runtime frameworks is unsafe). Some npm install/tar +// extraction paths don't preserve the executable bit on packed binaries, +// which makes node-pty fail with "posix_spawnp failed" the moment a terminal +// is created — reproduced on a plain `npm ci` in this repo's own CI. This is +// a no-op if the permission is already correct. +const fs = require("fs"); +const path = require("path"); + +const prebuildsDir = path.join(__dirname, "..", "node_modules", "node-pty", "prebuilds"); +if (fs.existsSync(prebuildsDir)) { + for (const platformDir of fs.readdirSync(prebuildsDir)) { + const helperPath = path.join(prebuildsDir, platformDir, "spawn-helper"); + if (fs.existsSync(helperPath)) { + fs.chmodSync(helperPath, 0o755); + console.log(`[fix-native-permissions] chmod +x ${path.relative(process.cwd(), helperPath)}`); + } + } +} diff --git a/app/src/agent-tools.test.ts b/app/src/agent-tools.test.ts index 354eaef..7d12426 100644 --- a/app/src/agent-tools.test.ts +++ b/app/src/agent-tools.test.ts @@ -3,6 +3,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { execSync } from "node:child_process"; +import { closeAll as closeAllTerminals } from "./terminal-manager"; import { readFile, writeFile, @@ -31,12 +32,16 @@ import { stopBackgroundCommand, listBackgroundCommands, killAllBackgroundCommands, + killBackgroundCommandsForWorkspace, httpRequest, findSymbolReferences, applyPatch, } from "./agent-tools"; -function waitFor(predicate: () => boolean, timeoutMs = 5000): Promise { +// 5000ms was tight enough that a slow spawn on Windows CI could still hit +// vitest's outer testTimeout (see app/vitest.config.ts) before this helper's +// own, more descriptive error had a chance to fire. +function waitFor(predicate: () => boolean, timeoutMs = 12_000): Promise { const start = Date.now(); return new Promise((resolve, reject) => { const check = () => { @@ -206,18 +211,24 @@ describe("agent-tools", () => { path.join(workspace, "package.json"), JSON.stringify({ scripts: { test: "vitest", build: "tsc" } }) ); - expect(detectProjectScripts(workspace)).toEqual({ test: "npm test", lint: undefined, format: undefined }); + expect(detectProjectScripts(workspace)).toEqual({ + test: "npm test", + lint: undefined, + format: undefined, + build: "npm run build", + }); }); - it("reports test, lint, and format scripts together when all are present", () => { + it("reports test, lint, format, and build scripts together when all are present", () => { fs.writeFileSync( path.join(workspace, "package.json"), - JSON.stringify({ scripts: { test: "vitest", lint: "eslint .", format: "prettier --write ." } }) + JSON.stringify({ scripts: { test: "vitest", lint: "eslint .", format: "prettier --write .", build: "tsc" } }) ); expect(detectProjectScripts(workspace)).toEqual({ test: "npm test", lint: "npm run lint", format: "npm run format", + build: "npm run build", }); }); }); @@ -379,6 +390,88 @@ describe("agent-tools", () => { const patch = ["--- /dev/null", "+++ b/../../etc/evil.txt", "@@ -0,0 +1,1 @@", "+pwned", ""].join("\n"); expect(() => applyPatch(workspace, patch)).toThrow(/outside the workspace/); }); + + // The patch base used to come from readFile(), which caps its result + // at MAX_READ_CHARS and appends a "[truncated ...]" marker — so + // patching a file past that cap silently destroyed its tail and wrote + // the marker into the source. + it("preserves content past the read-truncation cap", () => { + const body = Array.from({ length: 12_000 }, (_, i) => `line ${i}`).join("\n"); + const original = `first\n${body}\n`; + expect(original.length).toBeGreaterThan(100_000); + fs.writeFileSync(path.join(workspace, "big.txt"), original); + + const patch = ["--- a/big.txt", "+++ b/big.txt", "@@ -1,1 +1,1 @@", "-first", "+FIRST", ""].join("\n"); + applyPatch(workspace, patch); + + const updated = fs.readFileSync(path.join(workspace, "big.txt"), "utf-8"); + expect(updated).toBe(`FIRST\n${body}\n`); + expect(updated).not.toContain("truncated"); + }); + + // Writing as we went left earlier files modified when a later file in + // the same patch failed to align. + it("leaves every file untouched when one file in the patch fails to apply", () => { + fs.writeFileSync(path.join(workspace, "first.txt"), "alpha\n"); + fs.writeFileSync(path.join(workspace, "second.txt"), "actual content\n"); + const patch = [ + "--- a/first.txt", + "+++ b/first.txt", + "@@ -1,1 +1,1 @@", + "-alpha", + "+ALPHA", + "--- a/second.txt", + "+++ b/second.txt", + "@@ -1,1 +1,1 @@", + "-expected content", + "+new content", + "", + ].join("\n"); + + expect(() => applyPatch(workspace, patch)).toThrow(/Context mismatch/); + expect(fs.readFileSync(path.join(workspace, "first.txt"), "utf-8")).toBe("alpha\n"); + expect(fs.readFileSync(path.join(workspace, "second.txt"), "utf-8")).toBe("actual content\n"); + }); + + // Resolving every file before writing must not mean every file is read + // from disk first: two sections for one path have to chain, or the + // second silently discards the first. + it("chains two sections that touch the same file", () => { + fs.writeFileSync(path.join(workspace, "twice.txt"), "one\ntwo\n"); + const patch = [ + "--- a/twice.txt", + "+++ b/twice.txt", + "@@ -1,1 +1,1 @@", + "-one", + "+ONE", + "--- a/twice.txt", + "+++ b/twice.txt", + "@@ -2,1 +2,1 @@", + "-two", + "+TWO", + "", + ].join("\n"); + const result = applyPatch(workspace, patch); + expect(fs.readFileSync(path.join(workspace, "twice.txt"), "utf-8")).toBe("ONE\nTWO\n"); + expect(result.filesChanged).toEqual(["twice.txt"]); + }); + + it("patches a file the same diff created", () => { + const patch = [ + "--- /dev/null", + "+++ b/made.txt", + "@@ -0,0 +1,1 @@", + "+hello", + "--- a/made.txt", + "+++ b/made.txt", + "@@ -1,1 +1,1 @@", + "-hello", + "+HELLO", + "", + ].join("\n"); + applyPatch(workspace, patch); + expect(fs.readFileSync(path.join(workspace, "made.txt"), "utf-8")).toBe("HELLO"); + }); }); describe("executeTool", () => { @@ -392,6 +485,48 @@ describe("agent-tools", () => { }); }); + describe("terminal tools", () => { + afterEach(() => { + closeAllTerminals(); + }); + + function waitFor(predicate: () => boolean | Promise, timeoutMs = 12_000): Promise { + const start = Date.now(); + return new Promise((resolve, reject) => { + const check = async () => { + if (await predicate()) return resolve(); + if (Date.now() - start > timeoutMs) return reject(new Error("timed out waiting")); + setTimeout(check, 20); + }; + check(); + }); + } + + it("creates a terminal, writes a command to it, and reads the output back", async () => { + const created = (await executeTool(workspace, "create_terminal", { name: "test" })) as { id: string; name: string }; + expect(created.id).toBeTruthy(); + + await executeTool(workspace, "write_to_terminal", { terminal_id: created.id, input: "echo from-agent-terminal\r" }); + + await waitFor(async () => { + const output = (await executeTool(workspace, "read_terminal_output", { terminal_id: created.id })) as string; + return output.includes("from-agent-terminal"); + }); + const output = (await executeTool(workspace, "read_terminal_output", { terminal_id: created.id })) as string; + expect(output).toContain("from-agent-terminal"); + }); + + it("closes a terminal so further reads/writes fail", async () => { + const created = (await executeTool(workspace, "create_terminal", {})) as { id: string; name: string }; + await executeTool(workspace, "close_terminal", { terminal_id: created.id }); + await expect(executeTool(workspace, "read_terminal_output", { terminal_id: created.id })).rejects.toThrow(/No terminal/); + }); + + it("throws for an unknown terminal id", async () => { + await expect(executeTool(workspace, "write_to_terminal", { terminal_id: "nope", input: "x" })).rejects.toThrow(/No terminal/); + }); + }); + describe("run_command", () => { it("captures stdout and a zero exit code from a successful command", async () => { const output = await executeTool(workspace, "run_command", { command: "echo hello" }); @@ -494,6 +629,42 @@ describe("agent-tools", () => { for (let i = 0; i < 5; i++) startBackgroundCommand(workspace, "sleep 5", ".", `task-${i}`); expect(() => startBackgroundCommand(workspace, "sleep 5")).toThrow(/Already running/); }); + + it("kills and forgets only the tasks belonging to the given workspace, leaving other workspaces' tasks running", async () => { + const otherWorkspace = fs.mkdtempSync(path.join(os.tmpdir(), "agent-tools-test-other-")); + const { taskId: taskA } = startBackgroundCommand(workspace, "sleep 30"); + const { taskId: taskB } = startBackgroundCommand(otherWorkspace, "sleep 30"); + + const killedCount = killBackgroundCommandsForWorkspace(workspace); + + expect(killedCount).toBe(1); + // Killed AND removed from tracking — a task from a workspace you've + // switched away from shouldn't linger as a queryable id. + expect(() => getBackgroundOutput(taskA)).toThrow(/No background task/); + expect(getBackgroundOutput(taskB)).toContain("running"); + + // Clean up taskB now that the assertion above is done — leaving it + // running would leak a process for the rest of the suite, and on + // Windows, removing otherWorkspace while it's still that process's + // cwd fails with EBUSY (POSIX allows unlinking a directory a live + // process is using; Windows doesn't). Killing is fire-and-forget + // (killProcessTree doesn't wait for exit), so the directory removal + // is retried briefly to absorb the gap between signal and release. + killBackgroundCommandsForWorkspace(otherWorkspace); + for (let attempt = 0; attempt < 10; attempt++) { + try { + fs.rmSync(otherWorkspace, { recursive: true, force: true }); + break; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EBUSY" || attempt === 9) throw err; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + }); + + it("returns 0 when the workspace has no background tasks", () => { + expect(killBackgroundCommandsForWorkspace(workspace)).toBe(0); + }); }); describe("git tools", () => { @@ -527,6 +698,32 @@ describe("agent-tools", () => { expect(log).toContain("initial commit"); }); + // `git diff -- ""` and `git commit -m ""` are assembled + // into a string that ends up at `sh -c`. Double quotes do not stop the + // shell from expanding `$(...)`, so a model-supplied path or message + // used to be able to run arbitrary commands — including when the user + // had granted git_diff "always allow" as a read-only tool. + it("git_diff does not let a path argument reach the shell", async () => { + const marker = path.join(workspace, "diff-injection-marker"); + await gitDiff(workspace, false, `.$(touch ${marker})`); + expect(fs.existsSync(marker)).toBe(false); + }); + + it("git_commit does not let a commit message reach the shell", async () => { + const marker = path.join(workspace, "commit-injection-marker"); + await gitCommit(workspace, `initial $(touch ${marker})`); + expect(fs.existsSync(marker)).toBe(false); + }); + + it("git_commit preserves a message containing shell metacharacters", async () => { + const message = "fix: handle $HOME and `backticks` and 'quotes' and \"doubles\""; + await gitCommit(workspace, message); + const log = await gitLog(workspace, 5); + // The whole subject, not just its prefix — quoting that drops or + // mangles part of the message is as wrong as quoting that executes it. + expect(log).toContain(message); + }); + it("git_log returns nothing unusual with no commits yet", async () => { const output = await gitLog(workspace); expect(output).toContain("Exit code:"); diff --git a/app/src/agent-tools.ts b/app/src/agent-tools.ts index 3b879d1..14e1fa6 100644 --- a/app/src/agent-tools.ts +++ b/app/src/agent-tools.ts @@ -7,6 +7,12 @@ import { randomUUID } from "node:crypto"; import type { ToolDefinition } from "./providers/types"; import { getAccountToken } from "./accounts"; import { capturePageScreenshot } from "./browser-capture"; +import { killProcessTree } from "./process-tree"; +import { applySandbox, shellQuote } from "./command-sandbox"; +import { monitorProcess } from "./resource-monitor"; +import * as settingsStore from "./settings-store"; +import { resolveSafePath } from "./workspace-path"; +import * as terminalManager from "./terminal-manager"; const execAsync = promisify(exec); @@ -130,12 +136,13 @@ export const AGENT_TOOLS: ToolDefinition[] = [ { name: "run_command", description: - "Execute a shell command in the workspace (or a subdirectory of it) and return its stdout/stderr/exit code. Use for builds, tests, git, npm, etc. Commands that could affect the system outside the workspace (deleting elsewhere, shutting down the machine, privilege escalation, etc.) are rejected.", + "Execute a shell command in the workspace (or a subdirectory of it) and return its stdout/stderr/exit code. Use for builds, tests, git, npm, etc. Commands that could affect the system outside the workspace (deleting elsewhere, shutting down the machine, privilege escalation, etc.) are rejected. Runs inside an OS-level sandbox confined to the workspace where the platform supports it (Linux with bubblewrap installed, macOS always) — on Windows this containment isn't available and only the command-text checks above apply.", parameters: { type: "object", properties: { command: { type: "string", description: "The shell command to run." }, cwd: { type: "string", description: 'Working directory for the command, relative to the workspace root. Defaults to "."' }, + network: { type: "boolean", description: "Whether this command needs network access (e.g. npm install, curl). Defaults to false — most commands don't need it." }, }, required: ["command"], }, @@ -143,13 +150,14 @@ export const AGENT_TOOLS: ToolDefinition[] = [ { name: "run_code", description: - "Run a Python or JavaScript code snippet in the workspace and return its stdout/stderr/exit code. A convenience over run_command for multi-line code (no shell-quoting to worry about) — it is not a sandbox: the code runs with the same permissions as run_command and is subject to the same safety checks.", + "Run a Python or JavaScript code snippet in the workspace and return its stdout/stderr/exit code. A convenience over run_command for multi-line code (no shell-quoting to worry about) — subject to the same sandboxing (where available) and safety checks as run_command.", parameters: { type: "object", properties: { language: { type: "string", enum: ["python", "javascript"], description: "Which interpreter to run the code with." }, code: { type: "string", description: "The full source code to execute." }, cwd: { type: "string", description: 'Working directory, relative to the workspace root. Defaults to "."' }, + network: { type: "boolean", description: "Whether this code needs network access. Defaults to false." }, }, required: ["language", "code"], }, @@ -157,13 +165,14 @@ export const AGENT_TOOLS: ToolDefinition[] = [ { name: "start_background_command", description: - "Start a long-running command (dev server, build watcher, long test run) in the background and return immediately with a task id. Use get_background_output to check on it later and stop_background_command when done — unlike run_command, this doesn't block or time out. Subject to the same safety checks as run_command.", + "Start a long-running command (dev server, build watcher, long test run) in the background and return immediately with a task id. Use get_background_output to check on it later and stop_background_command when done — unlike run_command, this doesn't block or time out. Subject to the same safety checks and sandboxing as run_command.", parameters: { type: "object", properties: { command: { type: "string", description: "The shell command to run." }, cwd: { type: "string", description: 'Working directory, relative to the workspace root. Defaults to "."' }, name: { type: "string", description: "Short human-readable label for this task (e.g. \"dev server\")." }, + network: { type: "boolean", description: "Whether this command needs network access (e.g. a dev server that fetches data). Defaults to false." }, }, required: ["command"], }, @@ -195,6 +204,54 @@ export const AGENT_TOOLS: ToolDefinition[] = [ description: "List all background commands from this session with their status.", parameters: { type: "object", properties: {}, required: [] }, }, + { + name: "create_terminal", + description: + "Open a real interactive shell (a pseudo-terminal) in the workspace and return its id. Unlike run_command/start_background_command, this can be driven interactively over multiple turns — write_to_terminal to send input (including to a program already waiting for it, e.g. a REPL or a confirmation prompt), read_terminal_output to see what happened. Use this instead of start_background_command when you need to send input after the process has already started, not just read its output.", + parameters: { + type: "object", + properties: { + name: { type: "string", description: "Short human-readable label (e.g. \"debug session\")." }, + cwd: { type: "string", description: 'Working directory, relative to the workspace root. Defaults to "."' }, + }, + required: [], + }, + }, + { + name: "write_to_terminal", + description: "Send input to a terminal opened with create_terminal, as if typed at the keyboard. Include \\n (or \\r) to press Enter and actually run a typed command — without it, the text is typed but not submitted.", + parameters: { + type: "object", + properties: { + terminal_id: { type: "string", description: "The id returned by create_terminal." }, + input: { type: "string", description: "The text to send." }, + }, + required: ["terminal_id", "input"], + }, + }, + { + name: "read_terminal_output", + description: "Read the recent output of a terminal opened with create_terminal.", + parameters: { + type: "object", + properties: { + terminal_id: { type: "string", description: "The id returned by create_terminal." }, + tail_chars: { type: "number", description: "How many characters of recent output to return, from the end. Defaults to 4000." }, + }, + required: ["terminal_id"], + }, + }, + { + name: "close_terminal", + description: "Close a terminal opened with create_terminal, ending its shell process.", + parameters: { + type: "object", + properties: { + terminal_id: { type: "string", description: "The id returned by create_terminal." }, + }, + required: ["terminal_id"], + }, + }, { name: "git_status", description: "Show the working tree status (git status) for the workspace.", @@ -408,38 +465,6 @@ const MAX_COMMAND_OUTPUT_CHARS = 50_000; const COMMAND_TIMEOUT_MS = 60_000; const IGNORED_DIRS = new Set(["node_modules", ".git", "dist", "build", "out", "release", "__pycache__"]); -// Every tool call is confined to the chosen workspace directory — this -// resolves the (possibly relative, possibly attacker-crafted via a prompt -// injection in file content the model read) path and throws if it would -// escape that directory via ../ or an absolute path elsewhere on disk. -function resolveSafePath(workspaceRoot: string, relativePath: string): string { - const root = path.resolve(workspaceRoot); - const resolved = path.resolve(root, relativePath || "."); - const isWithin = (parent: string, child: string): boolean => { - const relative = path.relative(parent, child); - return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)); - }; - if (!isWithin(root, resolved)) { - throw new Error(`Path "${relativePath}" is outside the workspace directory.`); - } - - // Lexical checks alone are bypassable through a symlink inside the - // workspace that points elsewhere. Resolve the target, or its nearest - // existing parent for new files, and verify the real path too. - const realRoot = fs.realpathSync(root); - let existing = resolved; - while (!fs.existsSync(existing)) { - const parent = path.dirname(existing); - if (parent === existing) break; - existing = parent; - } - const realExisting = fs.realpathSync(existing); - if (!isWithin(realRoot, realExisting)) { - throw new Error(`Path "${relativePath}" resolves outside the workspace directory through a symbolic link.`); - } - return resolved; -} - export function readFile(workspaceRoot: string, relativePath: string, startLine?: number, endLine?: number): string { const target = resolveSafePath(workspaceRoot, relativePath); const stat = fs.statSync(target); @@ -783,17 +808,36 @@ function formatCommandResult(stdout: string, stderr: string, exitCode: number | return parts.join("\n\n"); } -export async function runCommand(workspaceRoot: string, command: string, relativeCwd = "."): Promise { +export async function runCommand( + workspaceRoot: string, + command: string, + relativeCwd = ".", + network = false +): Promise { const dangerReason = findDangerousCommandReason(command); if (dangerReason) throw new Error(dangerReason); const cwd = resolveSafePath(workspaceRoot, relativeCwd); + const wrappedCommand = applySandbox(command, { workspaceRoot, allowNetwork: network, cwd }); + const settings = settingsStore.getSettings(); + let stopMonitor = () => {}; try { - const { stdout, stderr } = await execAsync(command, { + const execPromise = execAsync(wrappedCommand, { cwd, timeout: COMMAND_TIMEOUT_MS, maxBuffer: 10 * 1024 * 1024, }); + // execPromise.child is a documented feature of promisify(exec) — the + // returned promise carries the underlying ChildProcess, which is the + // only way to get its pid for resource-monitor.ts to watch. + if (execPromise.child.pid) { + stopMonitor = monitorProcess( + execPromise.child.pid, + { maxMemoryMB: settings.sandboxMaxMemoryMB, maxCpuPercent: settings.sandboxMaxCpuPercent }, + () => execPromise.child.kill() + ); + } + const { stdout, stderr } = await execPromise; return formatCommandResult(stdout, stderr, 0); } catch (err) { const e = err as { stdout?: string; stderr?: string; code?: number; killed?: boolean; message: string }; @@ -801,6 +845,8 @@ export async function runCommand(workspaceRoot: string, command: string, relativ return `Command timed out after ${COMMAND_TIMEOUT_MS / 1000}s.\n\n${formatCommandResult(e.stdout ?? "", e.stderr ?? "", e.code ?? null)}`; } return formatCommandResult(e.stdout ?? "", e.stderr ?? e.message, e.code ?? null); + } finally { + stopMonitor(); } } @@ -814,7 +860,8 @@ export async function runCode( workspaceRoot: string, language: "python" | "javascript", code: string, - relativeCwd = "." + relativeCwd = ".", + network = false ): Promise { const dangerReason = findDangerousCommandReason(code); if (dangerReason) throw new Error(dangerReason); @@ -824,7 +871,7 @@ export async function runCode( fs.writeFileSync(tmpFile, code); try { const interpreter = language === "python" ? "python3" : "node"; - return await runCommand(workspaceRoot, `${interpreter} "${tmpFile}"`, relativeCwd); + return await runCommand(workspaceRoot, `${interpreter} "${tmpFile}"`, relativeCwd, network); } finally { fs.rmSync(tmpFile, { force: true }); } @@ -835,6 +882,10 @@ interface BackgroundTask { name: string; command: string; process: ChildProcess; + // Absolute path — lets killBackgroundCommandsForWorkspace() target only + // the tasks that belong to a workspace being switched away from, rather + // than every background task the process has ever started. + workspaceRoot: string; // Rolling tail of combined stdout+stderr — capped so a chatty dev server // can't grow memory unboundedly over a long session. output: string; @@ -851,7 +902,8 @@ export function startBackgroundCommand( workspaceRoot: string, command: string, relativeCwd = ".", - name?: string + name?: string, + network = false ): { taskId: string; name: string } { const dangerReason = findDangerousCommandReason(command); if (dangerReason) throw new Error(dangerReason); @@ -861,13 +913,25 @@ export function startBackgroundCommand( } const cwd = resolveSafePath(workspaceRoot, relativeCwd); - const child = spawn(command, { cwd, shell: true, stdio: ["ignore", "pipe", "pipe"] }); + const wrappedCommand = applySandbox(command, { workspaceRoot, allowNetwork: network, cwd }); + // detached so the shell becomes its own process group leader — lets + // killProcessTree() below signal the whole group (shell + whatever it + // spawned, e.g. `npm run dev` spawning `node`) instead of just the shell + // itself, which is all a plain .kill() would reach. No effect on Windows, + // where killProcessTree uses `taskkill /t` instead. + const child = spawn(wrappedCommand, { + cwd, + shell: true, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + }); const id = randomUUID().slice(0, 8); const task: BackgroundTask = { id, name: name?.trim() || command.slice(0, 40), command, process: child, + workspaceRoot: path.resolve(workspaceRoot), output: "", exitCode: null, startedAt: Date.now(), @@ -884,7 +948,16 @@ export function startBackgroundCommand( task.output += `\n[failed to start: ${err.message}]`; task.exitCode = -1; }); + const settings = settingsStore.getSettings(); + const stopMonitor = child.pid + ? monitorProcess( + child.pid, + { maxMemoryMB: settings.sandboxMaxMemoryMB, maxCpuPercent: settings.sandboxMaxCpuPercent }, + (reason) => append(Buffer.from(`\n[background task stopped: ${reason}]`)) + ) + : () => {}; child.on("exit", (code) => { + stopMonitor(); task.exitCode = code ?? -1; }); backgroundTasks.set(id, task); @@ -909,7 +982,7 @@ export function stopBackgroundCommand(taskId: string): string { const task = backgroundTasks.get(taskId); if (!task) throw new Error(`No background task with id "${taskId}".`); if (task.exitCode !== null) return `Task ${task.id} had already exited with code ${task.exitCode}.`; - task.process.kill(); + if (task.process.pid) killProcessTree(task.process.pid); return `Task ${task.id} (${task.name}) stopped.`; } @@ -924,11 +997,28 @@ export function listBackgroundCommands(): { id: string; name: string; command: s export function killAllBackgroundCommands(): void { for (const task of backgroundTasks.values()) { - if (task.exitCode === null) task.process.kill(); + if (task.exitCode === null && task.process.pid) killProcessTree(task.process.pid); } backgroundTasks.clear(); } +// Only tears down tasks belonging to the workspace being switched away from +// — background commands are otherwise never cleaned up until app quit +// (killAllBackgroundCommands, called from window-all-closed/before-quit), +// so switching to a different workspace mid-session used to leave the old +// one's tasks running indefinitely, silently eating into MAX_BACKGROUND_TASKS. +export function killBackgroundCommandsForWorkspace(workspaceRoot: string): number { + const root = path.resolve(workspaceRoot); + let killed = 0; + for (const [id, task] of backgroundTasks) { + if (task.workspaceRoot !== root) continue; + if (task.exitCode === null && task.process.pid) killProcessTree(task.process.pid); + backgroundTasks.delete(id); + killed++; + } + return killed; +} + function gitCommand(workspaceRoot: string, args: string): Promise { return runCommand(workspaceRoot, `git ${args}`, "."); } @@ -938,7 +1028,7 @@ export function gitStatus(workspaceRoot: string): Promise { } export function gitDiff(workspaceRoot: string, staged = false, relativePath?: string): Promise { - const target = relativePath ? ` -- "${relativePath}"` : ""; + const target = relativePath ? ` -- ${shellQuote(relativePath)}` : ""; return gitCommand(workspaceRoot, `diff${staged ? " --staged" : ""}${target}`); } @@ -948,7 +1038,9 @@ export function gitLog(workspaceRoot: string, count = 10): Promise { export async function gitCommit(workspaceRoot: string, message: string): Promise { await gitCommand(workspaceRoot, "add -A"); - return gitCommand(workspaceRoot, `commit -m ${JSON.stringify(message)}`); + // JSON.stringify escapes `"` and `\` but not `$` or backticks, and the + // result is handed to `sh -c` — so it is not a shell-quoting function. + return gitCommand(workspaceRoot, `commit -m ${shellQuote(message)}`); } const WEB_FETCH_TIMEOUT_MS = 15_000; @@ -1145,25 +1237,59 @@ function applyHunksToContent(content: string, hunks: PatchHunk[], filePath: stri return result.join("\n"); } +// Reads a file's complete content for use as the base of an edit. readFile() +// is display-oriented — it caps its result at MAX_READ_CHARS and appends a +// "[truncated ...]" marker — which makes it unsafe to write back: everything +// past the cap would be destroyed and the marker itself saved into the file. +function readFileForEdit(workspaceRoot: string, relativePath: string): string { + const target = resolveSafePath(workspaceRoot, relativePath); + if (fs.statSync(target).isDirectory()) throw new Error(`"${relativePath}" is a directory, not a file.`); + return fs.readFileSync(target, "utf-8"); +} + export function applyPatch(workspaceRoot: string, patchText: string): { filesChanged: string[] } { const files = parseUnifiedDiff(patchText); if (files.length === 0) throw new Error("No valid file patches found in the given diff."); - const filesChanged: string[] = []; + // Every file's outcome is resolved in memory before anything is written. + // Writing as we went meant a patch whose later file failed to align left + // the earlier ones already modified — a half-applied patch, which is the + // exact outcome this parser's refusal to fuzzy-match exists to avoid. + // + // `pending` (rather than a plain list) keeps resolution sequential: a diff + // with two sections for the same path, or one that creates a file and then + // patches it, has to see the earlier section's result instead of the stale + // copy on disk. null means the path is pending deletion. + const pending = new Map(); + const order: string[] = []; + const remember = (relativePath: string, content: string | null): void => { + if (!pending.has(relativePath)) order.push(relativePath); + pending.set(relativePath, content); + }; + for (const file of files) { if (file.newPath === null) { if (!file.oldPath) throw new Error("Patch deletes a file but its path (/dev/null on both sides) is missing."); - deletePath(workspaceRoot, file.oldPath, false); - filesChanged.push(file.oldPath); + remember(file.oldPath, null); continue; } - const isNewFile = file.oldPath === null; - const existingContent = isNewFile ? "" : readFile(workspaceRoot, file.newPath); - const newContent = applyHunksToContent(existingContent, file.hunks, file.newPath); - writeFile(workspaceRoot, file.newPath, newContent); - filesChanged.push(file.newPath); + let existingContent: string; + if (pending.has(file.newPath)) { + // Already touched by an earlier section of this same patch — a + // pending deletion reads back as empty, i.e. as a fresh file. + existingContent = pending.get(file.newPath) ?? ""; + } else { + existingContent = file.oldPath === null ? "" : readFileForEdit(workspaceRoot, file.newPath); + } + remember(file.newPath, applyHunksToContent(existingContent, file.hunks, file.newPath)); + } + + for (const relativePath of order) { + const content = pending.get(relativePath) ?? null; + if (content === null) deletePath(workspaceRoot, relativePath, false); + else writeFile(workspaceRoot, relativePath, content); } - return { filesChanged }; + return { filesChanged: order }; } export interface WebSearchResult { @@ -1225,11 +1351,14 @@ export interface ProjectScripts { test?: string; lint?: string; format?: string; + build?: string; } -// Backs the Test/Lint/Format quick-action buttons — only npm-style -// package.json scripts are recognized, which covers the JS/TS projects this -// app's Agent mode is primarily used against. +// Backs the Test/Lint/Format quick-action buttons, and (build + test) the +// default command list for the verification loop when the user hasn't set +// an explicit override — only npm-style package.json scripts are +// recognized, which covers the JS/TS projects this app's Agent mode is +// primarily used against. export function detectProjectScripts(workspaceRoot: string): ProjectScripts { const pkgPath = resolveSafePath(workspaceRoot, "package.json"); let scripts: Record = {}; @@ -1243,6 +1372,7 @@ export function detectProjectScripts(workspaceRoot: string): ProjectScripts { test: scripts.test ? "npm test" : undefined, lint: scripts.lint ? "npm run lint" : undefined, format: scripts.format ? "npm run format" : undefined, + build: scripts.build ? "npm run build" : undefined, }; } @@ -1324,7 +1454,25 @@ export async function githubReadFile(repository: string, filePath: string, ref?: return content.length > MAX_READ_CHARS ? `${content.slice(0, MAX_READ_CHARS)}\n\n[truncated]` : content; } +// Tools that reach the network — gated by settings.networkToolsEnabled as a +// baseline that's 100% enforceable on every platform (refusing to run the +// tool at all, rather than trying to block network access after the fact, +// which is what command-sandbox.ts's per-call `network` argument does for +// run_command/run_code/start_background_command instead). +const NETWORK_TOOLS = new Set([ + "web_search", + "fetch_url", + "http_request", + "capture_page_screenshot", + "github_list_repositories", + "github_repository_tree", + "github_read_file", +]); + export async function executeTool(workspaceRoot: string, name: string, args: Record): Promise { + if (NETWORK_TOOLS.has(name) && settingsStore.getSettings().networkToolsEnabled === false) { + throw new Error(`Network access for agent tools is turned off in Settings — "${name}" can't run.`); + } switch (name) { case "read_file": return readFile( @@ -1358,17 +1506,18 @@ export async function executeTool(workspaceRoot: string, name: string, args: Rec case "search_files": return searchFiles(workspaceRoot, String(args.query ?? ""), args.path ? String(args.path) : "."); case "run_command": - return runCommand(workspaceRoot, String(args.command ?? ""), args.cwd ? String(args.cwd) : "."); + return runCommand(workspaceRoot, String(args.command ?? ""), args.cwd ? String(args.cwd) : ".", args.network === true); case "run_code": { const language = args.language === "python" ? "python" : "javascript"; - return runCode(workspaceRoot, language, String(args.code ?? ""), args.cwd ? String(args.cwd) : "."); + return runCode(workspaceRoot, language, String(args.code ?? ""), args.cwd ? String(args.cwd) : ".", args.network === true); } case "start_background_command": return startBackgroundCommand( workspaceRoot, String(args.command ?? ""), args.cwd ? String(args.cwd) : ".", - args.name ? String(args.name) : undefined + args.name ? String(args.name) : undefined, + args.network === true ); case "get_background_output": return getBackgroundOutput(String(args.task_id ?? "")); @@ -1376,6 +1525,26 @@ export async function executeTool(workspaceRoot: string, name: string, args: Rec return stopBackgroundCommand(String(args.task_id ?? "")); case "list_background_commands": return listBackgroundCommands(); + case "create_terminal": + // No-op streaming callbacks: the model drives this by polling + // read_terminal_output rather than receiving push events — live + // streaming is reserved for the human-facing terminal panel, + // which goes through the dedicated terminal:create IPC channel + // instead of this tool-call path. + return terminalManager.createTerminal( + workspaceRoot, + { name: args.name ? String(args.name) : undefined, cwd: args.cwd ? String(args.cwd) : undefined }, + () => {}, + () => {} + ); + case "write_to_terminal": + terminalManager.writeToTerminal(String(args.terminal_id ?? ""), String(args.input ?? "")); + return { ok: true }; + case "read_terminal_output": + return terminalManager.readTerminalOutput(String(args.terminal_id ?? ""), typeof args.tail_chars === "number" ? args.tail_chars : undefined); + case "close_terminal": + terminalManager.closeTerminal(String(args.terminal_id ?? "")); + return { ok: true }; case "git_status": return gitStatus(workspaceRoot); case "git_diff": diff --git a/app/src/benchmark-runner.test.ts b/app/src/benchmark-runner.test.ts new file mode 100644 index 0000000..0b0775f --- /dev/null +++ b/app/src/benchmark-runner.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { approximateTokens, buildContextPrompt, contextTestSizes, runBenchmark, type BenchmarkChatExecutor } from "./benchmark-runner"; + +describe("benchmark helpers", () => { + it("builds bounded power-of-two context steps plus an exact cap", () => { + expect(contextTestSizes(10_000)).toEqual([2048, 4096, 8192, 10_000]); + expect(contextTestSizes(100)).toEqual([2048]); + }); + + it("creates approximately the requested prompt size", () => { + const prompt = buildContextPrompt(4096); + expect(approximateTokens(prompt)).toBeGreaterThan(4000); + expect(approximateTokens(prompt)).toBeLessThan(5200); + }); + + it("returns health, timing, comparison and context results", async () => { + const execute: BenchmarkChatExecutor = async (_provider, _model, messages, _options, onToken) => { + if (messages[0].content.startsWith("Context capacity")) return; + onToken({ done: false, message: { role: "assistant", content: "benchmark output" } }); + onToken({ done: true, usage: { promptTokens: 20, completionTokens: 4 } }); + }; + const result = await runBenchmark(execute, { + provider: "ollama", + model: "test", + maxContextLength: 4096, + compareCpuGpu: true, + }); + expect(result.health.healthy).toBe(true); + expect(result.primary?.completionTokens).toBe(4); + expect(result.comparison.supported).toBe(true); + expect(result.contextTests).toHaveLength(2); + }); +}); + diff --git a/app/src/benchmark-runner.ts b/app/src/benchmark-runner.ts new file mode 100644 index 0000000..e99c15b --- /dev/null +++ b/app/src/benchmark-runner.ts @@ -0,0 +1,294 @@ +import * as os from "node:os"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import type { ChatChunk, ChatMessage, ChatOptions, ProviderId } from "./providers/types"; + +const execFileAsync = promisify(execFile); +const SAMPLE_INTERVAL_MS = 250; + +export interface BenchmarkRequest { + provider: ProviderId; + model: string; + maxContextLength?: number; + outputTokens?: number; + compareCpuGpu?: boolean; +} + +export interface RuntimeHealthCheck { + healthy: boolean; + latencyMs: number; + error?: string; +} + +export interface ResourcePeaks { + peakSystemRamMB: number; + peakAppRamMB: number; + peakVramMB: number | null; + vramMeasurement: "nvidia-smi" | "rocm-smi" | "unavailable"; +} + +export interface InferenceMeasurement { + mode: "default" | "cpu" | "gpu"; + tokensPerSecond: number; + promptTokensPerSecond: number; + timeToFirstTokenMs: number; + totalTimeMs: number; + promptTokens: number; + completionTokens: number; + resources: ResourcePeaks; +} + +export interface ContextTestResult { + requestedTokens: number; + accepted: boolean; + elapsedMs: number; + error?: string; +} + +export interface BenchmarkResult { + schemaVersion: 1; + id: string; + createdAt: string; + provider: ProviderId; + model: string; + health: RuntimeHealthCheck; + primary: InferenceMeasurement | null; + comparison: { cpu?: InferenceMeasurement; gpu?: InferenceMeasurement; supported: boolean; note?: string }; + contextTests: ContextTestResult[]; + warnings: string[]; +} + +export type BenchmarkChatExecutor = ( + provider: ProviderId, + model: string, + messages: ChatMessage[], + options: ChatOptions, + onToken: (chunk: ChatChunk) => void, + signal?: AbortSignal +) => Promise; + +interface GpuSample { + usedMB: number | null; + source: ResourcePeaks["vramMeasurement"]; +} + +function round(value: number, digits = 1): number { + const scale = 10 ** digits; + return Math.round(value * scale) / scale; +} + +export function approximateTokens(text: string): number { + if (!text) return 0; + return Math.max(1, Math.ceil(text.length / 4)); +} + +export function contextTestSizes(maximum: number): number[] { + const cap = Math.max(2048, Math.min(131072, Math.floor(maximum))); + const sizes: number[] = []; + for (let size = 2048; size <= cap; size *= 2) sizes.push(size); + if (sizes[sizes.length - 1] !== cap) sizes.push(cap); + return sizes; +} + +export function buildContextPrompt(targetTokens: number): string { + // A repeated four-character token-like unit keeps allocation predictable. + // Provider tokenizers differ, so the result remains an approximation. + return `Context capacity test. Return only OK.\n${"test ".repeat(Math.max(1, targetTokens - 12))}`; +} + +async function sampleGpuMemory(): Promise { + try { + const { stdout } = await execFileAsync( + "nvidia-smi", + ["--query-compute-apps=used_memory", "--format=csv,noheader,nounits"], + { timeout: 2000, windowsHide: true } + ); + const values = stdout.split(/\r?\n/).map(Number).filter(Number.isFinite); + return { usedMB: values.length ? values.reduce((sum, value) => sum + value, 0) : 0, source: "nvidia-smi" }; + } catch { + // Continue with the ROCm probe below. + } + try { + const { stdout } = await execFileAsync("rocm-smi", ["--showmemuse", "--json"], { timeout: 2000, windowsHide: true }); + const parsed = JSON.parse(stdout) as Record>; + let usedMB = 0; + let found = false; + for (const card of Object.values(parsed)) { + for (const [key, raw] of Object.entries(card)) { + if (!/memory.*used/i.test(key)) continue; + const value = Number(String(raw).replace(/[^0-9.]/g, "")); + if (Number.isFinite(value)) { + usedMB += value > 1024 * 1024 ? value / 1024 / 1024 : value; + found = true; + } + } + } + return { usedMB: found ? usedMB : null, source: "rocm-smi" }; + } catch { + return { usedMB: null, source: "unavailable" }; + } +} + +async function startResourceSampler(): Promise<{ stop: () => Promise }> { + let stopped = false; + let peakSystemRamMB = 0; + let peakAppRamMB = 0; + let peakVramMB: number | null = null; + let vramMeasurement: ResourcePeaks["vramMeasurement"] = "unavailable"; + let pending = Promise.resolve(); + let gpuSampleInFlight = false; + const sample = () => { + const mem = process.memoryUsage(); + peakAppRamMB = Math.max(peakAppRamMB, mem.rss / 1024 / 1024); + peakSystemRamMB = Math.max(peakSystemRamMB, (os.totalmem() - os.freemem()) / 1024 / 1024); + if (gpuSampleInFlight) return; + gpuSampleInFlight = true; + pending = (async () => { + const gpu = await sampleGpuMemory(); + if (gpu.usedMB !== null) peakVramMB = Math.max(peakVramMB ?? 0, gpu.usedMB); + if (gpu.source !== "unavailable") vramMeasurement = gpu.source; + gpuSampleInFlight = false; + })(); + }; + sample(); + const timer = setInterval(sample, SAMPLE_INTERVAL_MS); + timer.unref(); + return { + async stop() { + if (!stopped) { + stopped = true; + clearInterval(timer); + sample(); + } + await pending; + return { + peakSystemRamMB: round(peakSystemRamMB), + peakAppRamMB: round(peakAppRamMB), + peakVramMB: peakVramMB === null ? null : round(peakVramMB), + vramMeasurement, + }; + }, + }; +} + +async function measureInference( + execute: BenchmarkChatExecutor, + request: BenchmarkRequest, + mode: InferenceMeasurement["mode"], + signal?: AbortSignal +): Promise { + const outputTokens = Math.max(8, Math.min(512, request.outputTokens ?? 96)); + const prompt = "Explain in concise technical terms why GPU memory bandwidth affects local language-model inference. Use complete sentences."; + const started = performance.now(); + let firstTokenAt: number | null = null; + let text = ""; + let reportedPromptTokens: number | undefined; + let reportedCompletionTokens: number | undefined; + const sampler = await startResourceSampler(); + const gpuLayers = mode === "cpu" ? 0 : mode === "gpu" ? 999 : undefined; + try { + await execute( + request.provider, + request.model, + [{ role: "user", content: prompt }], + { temperature: 0, maxTokens: outputTokens, seed: 42, gpuLayers }, + (chunk) => { + const piece = chunk.message?.content ?? ""; + if (piece && firstTokenAt === null) firstTokenAt = performance.now(); + text += piece; + reportedPromptTokens = chunk.usage?.promptTokens ?? reportedPromptTokens; + reportedCompletionTokens = chunk.usage?.completionTokens ?? reportedCompletionTokens; + }, + signal + ); + } catch (error) { + await sampler.stop(); + throw error; + } + const finished = performance.now(); + const resources = await sampler.stop(); + const promptTokens = reportedPromptTokens ?? approximateTokens(prompt); + const completionTokens = reportedCompletionTokens ?? approximateTokens(text); + const ttftMs = Math.max(0.1, (firstTokenAt ?? finished) - started); + const generationMs = Math.max(1, finished - (firstTokenAt ?? finished)); + return { + mode, + tokensPerSecond: round((completionTokens * 1000) / generationMs, 2), + promptTokensPerSecond: round((promptTokens * 1000) / ttftMs, 2), + timeToFirstTokenMs: round(ttftMs), + totalTimeMs: round(finished - started), + promptTokens, + completionTokens, + resources, + }; +} + +async function runContextTests( + execute: BenchmarkChatExecutor, + request: BenchmarkRequest, + signal?: AbortSignal +): Promise { + const results: ContextTestResult[] = []; + for (const requestedTokens of contextTestSizes(request.maxContextLength ?? 8192)) { + const started = performance.now(); + try { + await execute( + request.provider, + request.model, + [{ role: "user", content: buildContextPrompt(requestedTokens) }], + { temperature: 0, maxTokens: 1, contextLength: requestedTokens }, + () => undefined, + signal + ); + results.push({ requestedTokens, accepted: true, elapsedMs: round(performance.now() - started) }); + } catch (error) { + results.push({ requestedTokens, accepted: false, elapsedMs: round(performance.now() - started), error: (error as Error).message }); + break; + } + } + return results; +} + +export async function runBenchmark( + execute: BenchmarkChatExecutor, + request: BenchmarkRequest, + signal?: AbortSignal +): Promise { + const createdAt = new Date().toISOString(); + const result: BenchmarkResult = { + schemaVersion: 1, + id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + createdAt, + provider: request.provider, + model: request.model, + health: { healthy: false, latencyMs: 0 }, + primary: null, + comparison: { supported: false }, + contextTests: [], + warnings: [], + }; + const healthStarted = performance.now(); + try { + result.primary = await measureInference(execute, request, "default", signal); + result.health = { healthy: true, latencyMs: result.primary.timeToFirstTokenMs }; + } catch (error) { + result.health = { healthy: false, latencyMs: round(performance.now() - healthStarted), error: (error as Error).message }; + return result; + } + + if (request.compareCpuGpu) { + if (request.provider === "ollama" || request.provider === "llamacpp") { + result.comparison.supported = true; + result.comparison.cpu = await measureInference(execute, request, "cpu", signal); + result.comparison.gpu = await measureInference(execute, request, "gpu", signal); + } else { + result.comparison.note = "CPU/GPU layer control is available only for Ollama and the built-in llama.cpp runtime."; + } + } + result.contextTests = await runContextTests(execute, request, signal); + if (result.primary.resources.peakVramMB === null) { + result.warnings.push("VRAM usage was unavailable; install nvidia-smi or rocm-smi for measured VRAM peaks."); + } + result.warnings.push("Prompt-processing speed uses time-to-first-token and therefore includes model loading and request latency."); + return result; +} diff --git a/app/src/command-sandbox.test.ts b/app/src/command-sandbox.test.ts new file mode 100644 index 0000000..d6f7fd7 --- /dev/null +++ b/app/src/command-sandbox.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect } from "vitest"; +import * as path from "node:path"; +import { detectSandboxCapabilities, wrapCommand, applySandbox, shellQuote } from "./command-sandbox"; + +const has = (available: string[]) => (cmd: string) => available.includes(cmd); + +// wrapCommand resolves paths with the real (host-native) path.resolve +// regardless of the `platform` argument used to simulate capability +// detection — that argument only picks which sandbox mechanism to build for, +// it doesn't make path.resolve itself POSIX. So these tests build their +// expected paths the same way, via path.resolve, rather than hardcoding +// POSIX literals that would only match on Linux/macOS test runners and fail +// on Windows CI where path.resolve("/home/user/project") is +// "D:\\home\\user\\project". +const resolved = (p: string) => path.resolve(p); +// Mirrors buildMacSandboxProfile's own escaping of the resolved path so the +// expected profile string matches on any host, backslashes and all. +const macProfileEscaped = (p: string) => resolved(p).replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + +describe("detectSandboxCapabilities", () => { + it("prefers bubblewrap on Linux when available", () => { + expect(detectSandboxCapabilities("linux", has(["bwrap"]))).toEqual({ + filesystemConfinement: true, + networkDenial: true, + mechanism: "bubblewrap", + }); + }); + + it("reports no containment on Linux without bubblewrap", () => { + expect(detectSandboxCapabilities("linux", has([]))).toEqual({ + filesystemConfinement: false, + networkDenial: false, + mechanism: "none", + }); + }); + + it("uses sandbox-exec on macOS when available", () => { + expect(detectSandboxCapabilities("darwin", has(["sandbox-exec"]))).toEqual({ + filesystemConfinement: true, + networkDenial: true, + mechanism: "sandbox-exec", + }); + }); + + it("reports no containment on macOS without sandbox-exec", () => { + expect(detectSandboxCapabilities("darwin", has([]))).toEqual({ + filesystemConfinement: false, + networkDenial: false, + mechanism: "none", + }); + }); + + it("always reports no containment on Windows, regardless of PATH", () => { + expect(detectSandboxCapabilities("win32", has(["bwrap", "sandbox-exec"]))).toEqual({ + filesystemConfinement: false, + networkDenial: false, + mechanism: "none", + }); + }); +}); + +describe("wrapCommand", () => { + it("returns null when no sandbox mechanism is available", () => { + expect(wrapCommand("echo hi", { workspaceRoot: "/ws", allowNetwork: false }, "win32", has([]))).toBeNull(); + expect(wrapCommand("echo hi", { workspaceRoot: "/ws", allowNetwork: false }, "linux", has([]))).toBeNull(); + }); + + it("builds a bubblewrap invocation confining writes to the workspace, network denied by default", () => { + const wrapped = wrapCommand("npm test", { workspaceRoot: "/home/user/project", allowNetwork: false }, "linux", has(["bwrap"])); + expect(wrapped?.command).toBe("bwrap"); + expect(wrapped?.args).toContain("--unshare-all"); + expect(wrapped?.args).not.toContain("--share-net"); + expect(wrapped?.args).toEqual(expect.arrayContaining(["--bind", resolved("/home/user/project"), resolved("/home/user/project")])); + expect(wrapped?.args.slice(-3)).toEqual(["sh", "-c", "npm test"]); + }); + + it("adds --share-net to the bubblewrap invocation when network is explicitly allowed", () => { + const wrapped = wrapCommand("npm install", { workspaceRoot: "/home/user/project", allowNetwork: true }, "linux", has(["bwrap"])); + expect(wrapped?.args).toContain("--share-net"); + }); + + it("builds a sandbox-exec invocation with a profile scoped to the workspace", () => { + const wrapped = wrapCommand("npm test", { workspaceRoot: "/Users/me/project", allowNetwork: false }, "darwin", has(["sandbox-exec"])); + expect(wrapped?.command).toBe("sandbox-exec"); + expect(wrapped?.args[0]).toBe("-p"); + expect(wrapped?.args[1]).toContain(`(allow file-write* (subpath "${macProfileEscaped("/Users/me/project")}"))`); + expect(wrapped?.args[1]).toContain("(deny network*)"); + expect(wrapped?.args.slice(-3)).toEqual(["sh", "-c", "npm test"]); + }); + + it("allows network in the sandbox-exec profile when requested", () => { + const wrapped = wrapCommand("curl example.com", { workspaceRoot: "/Users/me/project", allowNetwork: true }, "darwin", has(["sandbox-exec"])); + expect(wrapped?.args[1]).toContain("(allow network*)"); + expect(wrapped?.args[1]).not.toContain("(deny network*)"); + }); + + // --chdir takes precedence over the working directory bwrap inherits from + // the spawning process, so pinning it to the workspace root silently + // overrode the subdirectory the caller had already resolved. + it("chdirs to the requested subdirectory, not just the workspace root", () => { + const wrapped = wrapCommand( + "npm test", + { workspaceRoot: "/home/user/project", allowNetwork: false, cwd: "/home/user/project/packages/api" }, + "linux", + has(["bwrap"]) + ); + const chdirIndex = wrapped!.args.indexOf("--chdir"); + expect(chdirIndex).toBeGreaterThan(-1); + expect(wrapped!.args[chdirIndex + 1]).toBe(resolved("/home/user/project/packages/api")); + // The workspace root is still what gets bound writable. + expect(wrapped!.args).toEqual(expect.arrayContaining(["--bind", resolved("/home/user/project"), resolved("/home/user/project")])); + }); + + it("falls back to the workspace root when no cwd is given", () => { + const wrapped = wrapCommand("npm test", { workspaceRoot: "/home/user/project", allowNetwork: false }, "linux", has(["bwrap"])); + const chdirIndex = wrapped!.args.indexOf("--chdir"); + expect(wrapped!.args[chdirIndex + 1]).toBe(resolved("/home/user/project")); + }); +}); + +describe("shellQuote", () => { + it("single-quotes for POSIX shells so substitutions stay inert", () => { + expect(shellQuote("simple.txt", "linux")).toBe("'simple.txt'"); + // The whole point: a POSIX shell expands these inside double quotes. + expect(shellQuote("$(id)", "linux")).toBe("'$(id)'"); + expect(shellQuote("`id`", "linux")).toBe("'`id`'"); + expect(shellQuote("it's", "linux")).toBe("'it'\\''s'"); + expect(shellQuote("", "linux")).toBe("''"); + }); + + // cmd.exe does not treat ' as a quote character, so POSIX quoting there + // would split ordinary arguments on their spaces instead of protecting + // them. It has no $(...) or backtick substitution to defend against. + it("double-quotes for cmd.exe, where single quotes are not quoting", () => { + expect(shellQuote("a message with spaces", "win32")).toBe('"a message with spaces"'); + expect(shellQuote('say "hi"', "win32")).toBe('"say ""hi"""'); + expect(shellQuote("a & b", "win32")).toBe('"a & b"'); + expect(shellQuote("", "win32")).toBe('""'); + }); +}); + +describe("applySandbox", () => { + it("returns the command unchanged when no sandbox mechanism is available", () => { + expect(applySandbox("echo hi", { workspaceRoot: "/ws", allowNetwork: false }, "win32", has(["bwrap"]))).toBe("echo hi"); + }); + + it("folds a bubblewrap-wrapped command into a single shell string ending in the original command", () => { + const result = applySandbox("npm test", { workspaceRoot: "/home/user/project", allowNetwork: false }, "linux", has(["bwrap"])); + expect(result.startsWith("'bwrap' ")).toBe(true); + expect(result).toContain(`'${resolved("/home/user/project")}'`); + expect(result.endsWith("'sh' '-c' 'npm test'")).toBe(true); + }); + + it("shell-quotes a workspace path containing a single quote so it can't break out of the wrapper", () => { + const result = applySandbox("echo hi", { workspaceRoot: "/home/user/it's-a-project", allowNetwork: false }, "linux", has(["bwrap"])); + // A raw unescaped single quote here would terminate the shell string + // early and let the rest of the path be interpreted as commands. + // applySandbox always quotes POSIX-style here since it's passed the + // simulated "linux" platform, regardless of the host OS actually + // running this test — only the resolved path's format varies by host. + expect(result).toContain(`'${resolved("/home/user/it's-a-project").replace(/'/g, "'\\''")}'`); + }); +}); diff --git a/app/src/command-sandbox.ts b/app/src/command-sandbox.ts new file mode 100644 index 0000000..97d74e9 --- /dev/null +++ b/app/src/command-sandbox.ts @@ -0,0 +1,208 @@ +import * as path from "node:path"; +import { execFileSync } from "node:child_process"; + +export interface SandboxCapabilities { + filesystemConfinement: boolean; + networkDenial: boolean; + // "unshare" is deliberately not offered as a mechanism even though the + // `unshare` binary itself is present on nearly every Linux system: + // creating a network namespace via plain `unshare --net` commonly + // requires privileges plain users don't have (fails with "Operation not + // permitted" on many stock kernel configs), which would make a sandboxed + // command fail outright instead of just running unsandboxed — worse than + // not attempting it. bubblewrap handles the unprivileged-namespace setup + // properly and is the only mechanism offered on Linux. + mechanism: "bubblewrap" | "sandbox-exec" | "none"; +} + +function commandExists(cmd: string): boolean { + try { + execFileSync(process.platform === "win32" ? "where" : "which", [cmd], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +// bwrap being on PATH isn't sufficient on its own — modern Ubuntu (23.10+) +// restricts unprivileged user-namespace creation by default (AppArmor +// policy), which makes bwrap fail at runtime with a permission error even +// when it's installed. A plain `which bwrap` check wouldn't catch that and +// would report false confidence, so this actually runs a trivial sandboxed +// no-op and checks it really works. +function canUseBubblewrap(): boolean { + if (!commandExists("bwrap")) return false; + try { + execFileSync("bwrap", ["--unshare-all", "--dev", "/dev", "--proc", "/proc", "true"], { + stdio: "ignore", + timeout: 5000, + }); + return true; + } catch { + return false; + } +} + +function defaultAvailabilityCheck(cmd: string): boolean { + return cmd === "bwrap" ? canUseBubblewrap() : commandExists(cmd); +} + +// `hasCommand` is injectable so tests can simulate "bwrap is/isn't usable" +// without actually shelling out. +export function detectSandboxCapabilities( + platform: NodeJS.Platform = process.platform, + hasCommand: (cmd: string) => boolean = defaultAvailabilityCheck +): SandboxCapabilities { + if (platform === "linux") { + if (hasCommand("bwrap")) return { filesystemConfinement: true, networkDenial: true, mechanism: "bubblewrap" }; + return { filesystemConfinement: false, networkDenial: false, mechanism: "none" }; + } + if (platform === "darwin") { + // Built into macOS — no install needed, so this should essentially + // always be available, but check anyway rather than assume. + if (hasCommand("sandbox-exec")) return { filesystemConfinement: true, networkDenial: true, mechanism: "sandbox-exec" }; + return { filesystemConfinement: false, networkDenial: false, mechanism: "none" }; + } + // Windows has no equivalent lightweight primitive: Windows Sandbox is a + // VM-like container requiring Pro/Enterprise, and Job Objects/restricted + // tokens don't confine the filesystem or network. Stays on the existing + // command-text blocklist plus resource-monitor.ts limits. + return { filesystemConfinement: false, networkDenial: false, mechanism: "none" }; +} + +export interface WrapCommandOptions { + workspaceRoot: string; + allowNetwork: boolean; + // Directory the command should start in. Defaults to the workspace root. + // bubblewrap is passed --chdir, which takes precedence over the working + // directory inherited from the spawning process, so a caller that resolved + // a subdirectory has to pass it through here — setting it only on the + // spawn options would be silently overridden. Callers are responsible for + // confining this to the workspace (they go through resolveSafePath); it is + // not re-validated here. + cwd?: string; +} + +export interface WrappedCommand { + command: string; + args: string[]; +} + +// Wraps `command` (a shell command string, run via `sh -c` either way) so it +// executes inside an OS-level sandbox instead of directly. Returns null when +// no sandboxing mechanism is available — callers should fall back to +// running `command` unwrapped rather than failing outright. +export function wrapCommand( + command: string, + opts: WrapCommandOptions, + platform: NodeJS.Platform = process.platform, + hasCommand: (cmd: string) => boolean = defaultAvailabilityCheck +): WrappedCommand | null { + const caps = detectSandboxCapabilities(platform, hasCommand); + const root = path.resolve(opts.workspaceRoot); + const startDir = opts.cwd ? path.resolve(opts.cwd) : root; + + if (caps.mechanism === "bubblewrap") { + const args = [ + "--ro-bind", + "/", + "/", + "--dev", + "/dev", + "--proc", + "/proc", + // Bound through (not a fresh --tmpfs) so files written to the + // real /tmp before the sandboxed process starts — e.g. run_code's + // temp script file, written via the host's os.tmpdir() — are + // still visible inside the sandbox. + "--bind", + "/tmp", + "/tmp", + "--bind", + root, + root, + "--chdir", + startDir, + "--die-with-parent", + "--unshare-all", + ]; + if (opts.allowNetwork) args.push("--share-net"); + args.push("sh", "-c", command); + return { command: "bwrap", args }; + } + + if (caps.mechanism === "sandbox-exec") { + return { command: "sandbox-exec", args: ["-p", buildMacSandboxProfile(root, opts.allowNetwork), "sh", "-c", command] }; + } + + return null; +} + +function buildMacSandboxProfile(workspaceRoot: string, allowNetwork: boolean): string { + const escaped = workspaceRoot.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + return [ + "(version 1)", + "(allow default)", + '(deny file-write* (subpath "/"))', + `(allow file-write* (subpath "${escaped}"))`, + // macOS's real temp dirs — os.tmpdir() resolves under + // /var/folders/.../T (symlinked from /private/var/folders), and + // /tmp itself is a symlink to /private/tmp. Both need to stay + // writable for the same reason /tmp does in the bubblewrap path + // above: run_code's temp script file lives there. + '(allow file-write* (subpath "/tmp"))', + '(allow file-write* (subpath "/private/tmp"))', + '(allow file-write* (subpath "/private/var/folders"))', + // The blanket file-write deny above also covers /dev — bubblewrap's + // Linux path gets a working /dev for free via --dev /dev, but + // sandbox-exec needs these listed explicitly. Without /dev/null + // specifically, ordinary tools that redirect to it internally + // (git being the one that surfaced this) fail with "could not open + // '/dev/null' for reading and writing: Operation not permitted". + '(allow file-write* (literal "/dev/null"))', + '(allow file-write* (literal "/dev/zero"))', + '(allow file-write* (literal "/dev/tty"))', + allowNetwork ? "(allow network*)" : "(deny network*)", + "", + ].join("\n"); +} + +// Quotes a single argument so the shell that will run it treats it as one +// literal value. Used both to fold a wrapped {command, args} back into the +// single shell-command string that `child_process.exec`/`spawn(..., +// {shell:true})` expect, and by agent-tools.ts when it builds a fixed command +// around a *value* the model supplied (a path for `git diff`, a message for +// `git commit`). +// +// The two shells need different treatment, and getting this wrong in either +// direction is a bug: +// +// - POSIX `sh`: single quotes, with an embedded quote written as '\''. +// Double quotes would not be enough, because `$(...)` and backticks are +// still expanded inside them. +// - Windows `cmd.exe`: single quotes are not quote characters at all, so +// POSIX quoting there would corrupt ordinary arguments rather than protect +// them. Double quotes are the right tool: `&`, `|`, `<` and `>` are +// literal inside them, and `$(...)`/backticks mean nothing to cmd.exe. +// An embedded double quote is written as "" — the convention both cmd.exe +// and the argv parser of the program being launched understand. +export function shellQuote(arg: string, platform: NodeJS.Platform = process.platform): string { + if (platform === "win32") return `"${arg.replace(/"/g, '""')}"`; + return `'${arg.replace(/'/g, `'\\''`)}'`; +} + +// Applies sandboxing to `command` if a mechanism is available on this +// platform, otherwise returns it unchanged — the one function agent-tools.ts +// actually calls before handing a command to exec/spawn. `platform`/ +// `hasCommand` are forwarded to wrapCommand purely so tests can exercise +// this without depending on the host OS. +export function applySandbox( + command: string, + opts: WrapCommandOptions, + platform: NodeJS.Platform = process.platform, + hasCommand: (cmd: string) => boolean = defaultAvailabilityCheck +): string { + const wrapped = wrapCommand(command, opts, platform, hasCommand); + if (!wrapped) return command; + return [wrapped.command, ...wrapped.args].map((arg) => shellQuote(arg, platform)).join(" "); +} diff --git a/app/src/download-jobs-store.test.ts b/app/src/download-jobs-store.test.ts new file mode 100644 index 0000000..bcd1b9c --- /dev/null +++ b/app/src/download-jobs-store.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from "vitest"; +import { listJobs, getJob, createJob, updateJob, deleteJob, type DownloadShard } from "./download-jobs-store"; + +function shard(overrides: Partial = {}): DownloadShard { + return { + filename: "model.gguf", + path: "/models/model.gguf", + expectedBytes: 1000, + receivedBytes: 0, + state: "queued", + ...overrides, + }; +} + +describe("download-jobs-store", () => { + it("starts with an empty list", () => { + expect(listJobs()).toEqual([]); + }); + + it("creates a job with generated id, default state, and timestamps", () => { + const job = createJob({ + kind: "huggingface", + modelName: "Test Model", + publisher: "test-org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "test-org/test-model", + shards: [shard()], + }); + expect(job.id).toBeTruthy(); + expect(job.state).toBe("queued"); + expect(job.retryCount).toBe(0); + expect(job.createdAt).toBeTruthy(); + expect(job.updatedAt).toBe(job.createdAt); + expect(listJobs().map((j) => j.id)).toContain(job.id); + }); + + it("round-trips a job through getJob", () => { + const created = createJob({ + kind: "huggingface", + modelName: "Another Model", + publisher: "org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "org/another-model", + shards: [shard()], + }); + expect(getJob(created.id)).toEqual(created); + }); + + it("returns null for an unknown job id", () => { + expect(getJob("does-not-exist")).toBeNull(); + }); + + it("updates state, error, retryCount, and shards while bumping updatedAt", async () => { + const created = createJob({ + kind: "huggingface", + modelName: "M", + publisher: "org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "org/m", + shards: [shard()], + }); + await new Promise((r) => setTimeout(r, 5)); + const updatedShards = [shard({ receivedBytes: 500 })]; + const updated = updateJob(created.id, { state: "downloading", shards: updatedShards, retryCount: 1 }); + expect(updated?.state).toBe("downloading"); + expect(updated?.shards).toEqual(updatedShards); + expect(updated?.retryCount).toBe(1); + expect(updated?.updatedAt).not.toBe(created.updatedAt); + }); + + it("returns null when updating an unknown job", () => { + expect(updateJob("does-not-exist", { state: "ready" })).toBeNull(); + }); + + it("sets a typed error with kind and retryable flags", () => { + const created = createJob({ + kind: "huggingface", + modelName: "M", + publisher: "org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "org/m", + shards: [shard()], + }); + const updated = updateJob(created.id, { + state: "failed", + error: { message: "Gated repo — accept the license first.", kind: "license_required", retryable: false }, + }); + expect(updated?.error).toEqual({ message: "Gated repo — accept the license first.", kind: "license_required", retryable: false }); + }); + + it("deletes a job", () => { + const created = createJob({ + kind: "huggingface", + modelName: "M", + publisher: "org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "org/m", + shards: [shard()], + }); + deleteJob(created.id); + expect(getJob(created.id)).toBeNull(); + }); + + it("deleting an unknown job is a harmless no-op", () => { + expect(() => deleteJob("does-not-exist")).not.toThrow(); + }); +}); diff --git a/app/src/download-jobs-store.ts b/app/src/download-jobs-store.ts new file mode 100644 index 0000000..558751e --- /dev/null +++ b/app/src/download-jobs-store.ts @@ -0,0 +1,115 @@ +import * as path from "node:path"; +import { randomUUID } from "node:crypto"; +import { app } from "electron"; +import { readJson, writeJson } from "./json-store"; + +export type DownloadJobState = + | "queued" + | "resolving" + | "downloading" + | "paused" + | "verifying" + | "installing" + | "ready" + | "failed" + | "cancelled"; + +export type DownloadErrorKind = + | "auth_required" + | "license_required" + | "not_found" + | "disk_space" + | "permission" + | "verification_failed" + | "network" + | "unknown"; + +export interface DownloadJobError { + message: string; + kind: DownloadErrorKind; + retryable: boolean; +} + +export interface DownloadShard { + filename: string; + path: string; + expectedBytes: number; + receivedBytes: number; + sha256?: string; + etag?: string; + state: DownloadJobState; +} + +export interface DownloadJob { + id: string; + kind: "huggingface" | "ollama"; + modelName: string; + publisher: string; + quantization?: string; + backend: "llamacpp" | "mlx" | "vllm" | "ollama" | "transformers"; + destinationDir: string; + modelId: string; + shards: DownloadShard[]; + state: DownloadJobState; + error?: DownloadJobError; + retryCount: number; + jobReceivedBytes?: number; + totalBytes?: number; + bytesPerSecond?: number; + etaSeconds?: number; + recoveredAtStartup?: boolean; + createdAt: string; + updatedAt: string; +} + +function filePath(): string { + return path.join(app.getPath("userData"), "download-jobs.json"); +} + +export function listJobs(): DownloadJob[] { + return readJson(filePath(), []); +} + +export function getJob(id: string): DownloadJob | null { + return listJobs().find((j) => j.id === id) ?? null; +} + +export function createJob( + partial: Pick & + Partial> +): DownloadJob { + const now = new Date().toISOString(); + const job: DownloadJob = { + id: randomUUID(), + state: "queued", + retryCount: 0, + createdAt: now, + updatedAt: now, + ...partial, + }; + const all = listJobs(); + all.push(job); + writeJson(filePath(), all); + return job; +} + +// Callers pass whole replacement `shards` arrays rather than patching one +// shard in place — the queue worker always has the full up-to-date shard +// list in memory already (it's the one mutating it), so round-tripping a +// single shard's delta through this API would just add complexity for no +// benefit. +export function updateJob( + id: string, + partial: Partial> +): DownloadJob | null { + const all = listJobs(); + const idx = all.findIndex((j) => j.id === id); + if (idx === -1) return null; + all[idx] = { ...all[idx], ...partial, updatedAt: new Date().toISOString() }; + writeJson(filePath(), all); + return all[idx]; +} + +export function deleteJob(id: string): void { + writeJson(filePath(), listJobs().filter((j) => j.id !== id)); +} diff --git a/app/src/download-queue.test.ts b/app/src/download-queue.test.ts new file mode 100644 index 0000000..d88387a --- /dev/null +++ b/app/src/download-queue.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect, vi } from "vitest"; +import { createJob, updateJob, listJobs, type DownloadShard, type DownloadJobState } from "./download-jobs-store"; +import { init, broadcast, resumeInterruptedJobs } from "./download-queue"; + +function shard(overrides: Partial = {}): DownloadShard { + return { + filename: "model.gguf", + path: "/models/model.gguf", + expectedBytes: 1000, + receivedBytes: 0, + state: "queued", + ...overrides, + }; +} + +function makeJob(state: DownloadJobState) { + const job = createJob({ + kind: "huggingface", + modelName: "M", + publisher: "org", + backend: "llamacpp", + destinationDir: "/models", + modelId: "org/m", + shards: [shard()], + }); + updateJob(job.id, { state }); + return job.id; +} + +describe("download-queue: broadcast plumbing", () => { + it("does nothing when no window has been registered via init()", () => { + expect(() => broadcast()).not.toThrow(); + }); + + it("sends the current job list to the registered window's webContents on broadcast", () => { + const send = vi.fn(); + init(() => ({ webContents: { send } }) as never); + broadcast(); + expect(send).toHaveBeenCalledWith("downloads:update", listJobs()); + }); +}); + +describe("resumeInterruptedJobs", () => { + it("requeues jobs stuck in downloading or resolving back to queued", () => { + const downloadingId = makeJob("downloading"); + const resolvingId = makeJob("resolving"); + + resumeInterruptedJobs(); + + expect(listJobs().find((j) => j.id === downloadingId)?.state).toBe("queued"); + expect(listJobs().find((j) => j.id === resolvingId)?.state).toBe("queued"); + }); + + it("leaves jobs in paused, ready, failed, or cancelled untouched", () => { + const pausedId = makeJob("paused"); + const readyId = makeJob("ready"); + + resumeInterruptedJobs(); + + expect(listJobs().find((j) => j.id === pausedId)?.state).toBe("paused"); + expect(listJobs().find((j) => j.id === readyId)?.state).toBe("ready"); + }); +}); diff --git a/app/src/download-queue.ts b/app/src/download-queue.ts new file mode 100644 index 0000000..4d52404 --- /dev/null +++ b/app/src/download-queue.ts @@ -0,0 +1,107 @@ +import type { BrowserWindow } from "electron"; +import * as downloadWorker from "./download-worker"; +import * as fs from "node:fs"; +import { createJob, deleteJob, getJob, listJobs, updateJob, type DownloadJob } from "./download-jobs-store"; +import { getDownloadManager } from "./native-downloader"; +import { detectModelFormat, getSpecs, resolveAutomaticRuntime } from "./system-specs"; + +export interface DownloadControls { concurrency: number; bandwidthMbps: number } +export interface RecoveryStatus { recoveredJobs: number; recoveredAt: string | null } +let recoveryStatus: RecoveryStatus = { recoveredJobs: 0, recoveredAt: null }; +export function getJobs(): DownloadJob[] { return listJobs(); } + +// Deliberately a getWindow() closure, not a captured event.sender from one +// ipcMain.handle invocation — jobs push progress at times disconnected from +// any single IPC call (a download keeps running long after the request that +// started it returned), and the window that started a job may have since +// been closed/reopened (e.g. macOS activate-with-no-windows). Mirrors the +// same pattern menu.ts and updater.ts already use for this exact reason. +let windowGetter: () => BrowserWindow | null = () => null; + +export function init(getWindow: () => BrowserWindow | null): void { + windowGetter = getWindow; +} + +// The one broadcast channel in the app that isn't scoped to a requestId — +// every open window gets the full current job list whenever anything +// changes, so the download center reflects live state regardless of which +// page is open or when it mounted. +export function broadcast(): void { + windowGetter()?.webContents.send("downloads:update", listJobs()); +} + +// Called once at app startup. A job left in "downloading" or "resolving" +// can only mean the app was killed mid-download last session — not a +// deliberate pause — so it goes back to "queued" for the worker to pick up +// again (the underlying .part file and its recorded progress are untouched; +// resuming is exactly what downloadGgufFile's Range-request logic already +// does for a partial file, whatever caused the interruption). +export function resumeInterruptedJobs(): void { + let recoveredJobs = 0; + for (const job of listJobs()) { + if (["downloading", "resolving", "verifying", "installing"].includes(job.state)) { + updateJob(job.id, { state: "queued", recoveredAtStartup: true }); + recoveredJobs++; + } + } + broadcast(); + downloadWorker.wake(); + recoveryStatus = { recoveredJobs, recoveredAt: new Date().toISOString() }; +} + +export function getRecoveryStatus(): RecoveryStatus { return recoveryStatus; } + +export function configure(controls: DownloadControls): DownloadControls { + const concurrency = Math.max(1, Math.min(8, Math.floor(controls.concurrency))); + const bandwidthMbps = Math.max(0, Number(controls.bandwidthMbps) || 0); + const manager = getDownloadManager(); + manager.setGlobalConcurrency(concurrency); + manager.setBandwidthLimit(bandwidthMbps > 0 ? bandwidthMbps * 1024 * 1024 / 8 : undefined); + return { concurrency, bandwidthMbps }; +} + +export async function createHuggingFaceJob(input: { modelId: string; filename: string; expectedBytes: number; destinationDir: string; backend?: "automatic" | DownloadJob["backend"] }): Promise { + const filename = input.filename.replace(/\\/g, "/").split("/").pop() || input.filename; + // "automatic" (the default) resolves the same way the model-recommendation + // flow does: format inferred from the filename/repo id, matched against + // this machine's detected hardware. + const backend = !input.backend || input.backend === "automatic" + ? resolveAutomaticRuntime(detectModelFormat(input.filename) === "unknown" ? detectModelFormat(input.modelId) : detectModelFormat(input.filename), await getSpecs()) + : input.backend; + const job = createJob({ + kind: "huggingface", modelName: input.modelId.split("/").pop() || input.modelId, + publisher: input.modelId.split("/")[0] || "Hugging Face", backend, + destinationDir: input.destinationDir, modelId: input.modelId, + quantization: filename.match(/(Q\d(?:_[A-Z0-9]+)*)/i)?.[1], + shards: [{ filename: input.filename, path: `${input.destinationDir}/${filename}`, expectedBytes: Math.max(0, input.expectedBytes), receivedBytes: 0, state: "queued" }], + }); + broadcast(); downloadWorker.wake(); return job; +} + +export function pauseJob(id: string): void { + const job = getJob(id); if (!job || !["queued", "resolving", "downloading"].includes(job.state)) return; + downloadWorker.pause(id); updateJob(id, { state: "paused", etaSeconds: undefined, bytesPerSecond: undefined }); broadcast(); +} +export function resumeJob(id: string): void { + const job = getJob(id); if (!job || !["paused", "failed", "cancelled"].includes(job.state)) return; + downloadWorker.clearRetryTimer(id); updateJob(id, { state: "queued", error: undefined, recoveredAtStartup: false }); broadcast(); downloadWorker.wake(); +} +export function retryJob(id: string): void { resumeJob(id); } +export function cancelJob(id: string): void { + const job = getJob(id); if (!job || ["ready", "cancelled"].includes(job.state)) return; + downloadWorker.cancel(id); updateJob(id, { state: "cancelled", etaSeconds: undefined, bytesPerSecond: undefined }); broadcast(); +} +export function removeJob(id: string): void { + const job = getJob(id); if (!job) return; + downloadWorker.cancel(id); + for (const shard of job.shards) { + for (const suffix of [".part", ".part.json"]) { try { fs.rmSync(`${shard.path}${suffix}`, { force: true }); } catch { /* best effort */ } } + } + deleteJob(id); broadcast(); +} +export function diskForecast(id: string): { requiredBytes: number; availableBytes: number | null; enough: boolean | null } { + const job = getJob(id); if (!job) throw new Error("Download job not found"); + const requiredBytes = job.shards.reduce((sum, shard) => sum + Math.max(0, shard.expectedBytes - shard.receivedBytes), 0); + try { const stats = fs.statfsSync(job.destinationDir); const availableBytes = stats.bavail * stats.bsize; return { requiredBytes, availableBytes, enough: availableBytes >= requiredBytes }; } + catch { return { requiredBytes, availableBytes: null, enough: null }; } +} diff --git a/app/src/download-verification.test.ts b/app/src/download-verification.test.ts new file mode 100644 index 0000000..174edd8 --- /dev/null +++ b/app/src/download-verification.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { createHash } from "node:crypto"; +import { computeSha256, hasGgufMagic, getDiskSpace } from "./download-verification"; + +describe("computeSha256", () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-verification-test-")); + }); + + it("matches Node's own synchronous hash of the same content", async () => { + const file = path.join(dir, "data.bin"); + const content = Buffer.from("hello world".repeat(1000)); + fs.writeFileSync(file, content); + const expected = createHash("sha256").update(content).digest("hex"); + expect(await computeSha256(file)).toBe(expected); + }); + + it("rejects when the file doesn't exist", async () => { + await expect(computeSha256(path.join(dir, "missing.bin"))).rejects.toThrow(); + }); +}); + +describe("hasGgufMagic", () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-verification-test-")); + }); + + it("returns true for a file starting with the GGUF magic bytes", () => { + const file = path.join(dir, "model.gguf"); + fs.writeFileSync(file, Buffer.concat([Buffer.from("GGUF"), Buffer.from([3, 0, 0, 0])])); + expect(hasGgufMagic(file)).toBe(true); + }); + + it("returns false for a file without the magic bytes (e.g. an HTML error page)", () => { + const file = path.join(dir, "not-a-model.gguf"); + fs.writeFileSync(file, "404"); + expect(hasGgufMagic(file)).toBe(false); + }); + + it("returns false for a file too short to contain the magic bytes", () => { + const file = path.join(dir, "tiny.gguf"); + fs.writeFileSync(file, "GG"); + expect(hasGgufMagic(file)).toBe(false); + }); + + it("returns false when the file doesn't exist", () => { + expect(hasGgufMagic(path.join(dir, "missing.gguf"))).toBe(false); + }); +}); + +describe("getDiskSpace", () => { + it("reports positive free and total bytes for a real directory", () => { + const { freeBytes, totalBytes } = getDiskSpace(os.tmpdir()); + expect(totalBytes).toBeGreaterThan(0); + expect(freeBytes).toBeGreaterThan(0); + expect(freeBytes).toBeLessThanOrEqual(totalBytes); + }); +}); diff --git a/app/src/download-verification.ts b/app/src/download-verification.ts new file mode 100644 index 0000000..aba1327 --- /dev/null +++ b/app/src/download-verification.ts @@ -0,0 +1,53 @@ +import * as fs from "node:fs"; +import { createHash } from "node:crypto"; + +// Streamed, not buffered — a 40GB model shard must never be loaded whole +// into memory just to hash it, same discipline the download itself already +// follows (see huggingface.ts's downloadGgufFile). +export function computeSha256(filePath: string): Promise { + return new Promise((resolve, reject) => { + const hash = createHash("sha256"); + const stream = fs.createReadStream(filePath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + stream.on("error", reject); + }); +} + +const GGUF_MAGIC = Buffer.from("GGUF", "ascii"); + +// A file that's the right size and even the right checksum could still be +// garbage if something upstream served an HTML error page sized to match +// (rare, but a Range-into-a-CDN-error-page is a real failure mode) — a +// cheap, independent sanity check on top of the checksum, not a substitute +// for it. +export function hasGgufMagic(filePath: string): boolean { + let fd: number; + try { + fd = fs.openSync(filePath, "r"); + } catch { + return false; + } + try { + const buf = Buffer.alloc(4); + const bytesRead = fs.readSync(fd, buf, 0, 4, 0); + return bytesRead === 4 && buf.equals(GGUF_MAGIC); + } finally { + fs.closeSync(fd); + } +} + +export interface DiskSpace { + freeBytes: number; + totalBytes: number; +} + +// Node 18.15+'s fs.statfsSync — no new dependency needed. `dir` must exist; +// callers resolve/create the destination directory before checking. +export function getDiskSpace(dir: string): DiskSpace { + const stats = fs.statfsSync(dir); + return { + freeBytes: stats.bavail * stats.bsize, + totalBytes: stats.blocks * stats.bsize, + }; +} diff --git a/app/src/download-worker.ts b/app/src/download-worker.ts new file mode 100644 index 0000000..cc8b916 --- /dev/null +++ b/app/src/download-worker.ts @@ -0,0 +1,166 @@ +import * as accounts from "./accounts"; +import * as downloadQueue from "./download-queue"; +import { getJob, listJobs, updateJob, type DownloadErrorKind, type DownloadJob, type DownloadShard } from "./download-jobs-store"; +import { logger } from "./logger"; +import { getDownloadManager, type JobEvent, type JsDownloadJob } from "./native-downloader"; + +const MAX_AUTO_RETRIES = 3; + +// How often raw JobEvents get coalesced into a `downloads:update` broadcast +// — matches the ~100ms cadence the Rust side's own progress ticker already +// uses, so this never becomes the bottleneck. A job's terminal state always +// broadcasts immediately regardless (see `driveJob`'s `finally`), so the UI +// never has to wait out this throttle to see a job actually finish. +const BROADCAST_THROTTLE_MS = 100; +let lastBroadcastAt = 0; + +function throttledBroadcast(): void { + const now = Date.now(); + if (now - lastBroadcastAt < BROADCAST_THROTTLE_MS) return; + lastBroadcastAt = now; + downloadQueue.broadcast(); +} + +// Job ids currently being driven — guards against `wake()` picking up the +// same "queued" job twice; the real concurrency gate is the Rust manager's +// own semaphore, this is purely a JS-side duplicate-start guard. +const active = new Set(); +const retryTimers = new Map(); + +export function clearRetryTimer(jobId: string): void { + const timer = retryTimers.get(jobId); + if (timer) clearTimeout(timer); + retryTimers.delete(jobId); +} + +function toJsDownloadJob(job: DownloadJob): JsDownloadJob { + return { + id: job.id, + modelId: job.modelId, + token: accounts.getAccountToken("huggingface") ?? undefined, + destinationDir: job.destinationDir, + shards: job.shards.map((s) => ({ + filename: s.filename, + path: s.path, + expectedBytes: s.expectedBytes, + receivedBytes: s.receivedBytes, + sha256: s.sha256, + })), + }; +} + +function withShardPatch(job: DownloadJob, filename: string, patch: Partial): DownloadShard[] { + return job.shards.map((s) => (s.filename === filename ? { ...s, ...patch } : s)); +} + +function handleEvent(event: JobEvent): void { + const job = getJob(event.jobId); + if (!job) return; // job was deleted while its download was in flight + + switch (event.kind) { + case "shard_state": + if (event.shardFilename && event.shardState) { + updateJob(job.id, { shards: withShardPatch(job, event.shardFilename, { state: event.shardState as DownloadShard["state"] }) }); + } + break; + case "shard_progress": + if (event.shardFilename && event.receivedBytes !== undefined) { + updateJob(job.id, { + shards: withShardPatch(job, event.shardFilename, { receivedBytes: event.receivedBytes }), + jobReceivedBytes: event.jobReceivedBytes, + totalBytes: event.totalBytes, + bytesPerSecond: event.bytesPerSec, + etaSeconds: event.etaSeconds, + }); + } + break; + case "job_state": + if (event.jobState) { + updateJob(job.id, { state: event.jobState as DownloadJob["state"] }); + } + break; + case "job_error": + updateJob(job.id, { + state: "failed", + error: { + message: event.errorMessage ?? "Download failed.", + kind: (event.errorKind ?? "unknown") as DownloadErrorKind, + retryable: event.retryable ?? false, + }, + }); + break; + } +} + +// A `"failed"` job whose error is marked retryable gets a small number of +// automatic requeue attempts with exponential backoff before it's left for +// a manual retry — distinct from the Rust side's own per-chunk retry +// (MAX_ATTEMPTS_PER_CHUNK), which handles much shorter-lived transient +// failures within a single shard transfer. +function maybeAutoRetry(jobId: string): void { + const job = getJob(jobId); + if (!job || job.state !== "failed" || !job.error?.retryable || job.retryCount >= MAX_AUTO_RETRIES) return; + + const backoffMs = 1000 * 2 ** job.retryCount; + if (retryTimers.has(jobId)) return; + const expectedRetryCount = job.retryCount; + const timer = setTimeout(() => { + retryTimers.delete(jobId); + const current = getJob(jobId); + if (!current || current.state !== "failed" || !current.error?.retryable || current.retryCount !== expectedRetryCount) return; + updateJob(jobId, { state: "queued", retryCount: expectedRetryCount + 1 }); + wake(); + }, backoffMs); + timer.unref(); + retryTimers.set(jobId, timer); +} + +async function driveJob(job: DownloadJob): Promise { + active.add(job.id); + try { + await getDownloadManager().startJob(toJsDownloadJob(job), (err, event) => { + if (err) { + logger.error(`Download job ${job.id}: event callback error: ${err.message}`); + return; + } + handleEvent(event); + throttledBroadcast(); + }); + } catch (err) { + // The rejection carries the same message the "job_error" event + // already recorded via handleEvent — this catch exists only so an + // unexpected throw here doesn't become an unhandled rejection. + logger.error(`Download job ${job.id}: ${(err as Error).message}`); + } finally { + active.delete(job.id); + downloadQueue.broadcast(); + maybeAutoRetry(job.id); + } +} + +// Picks up every currently-"queued" Hugging Face job not already being +// driven. Safe to call repeatedly/redundantly — `active` prevents double +// starts. Called once at app-ready and again whenever something might have +// made a "queued" job appear (a fresh `downloads:create`, a manual +// retry/resume, or `download-queue.ts`'s startup requeue). +export function wake(): void { + for (const job of listJobs()) { + if (job.state === "queued" && job.kind === "huggingface" && !active.has(job.id)) { + void driveJob(job); + } + } +} + +export function start(): void { + wake(); +} + +export function pause(jobId: string): void { + clearRetryTimer(jobId); + getDownloadManager().pauseJob(jobId); +} + +export function cancel(jobId: string): void { + clearRetryTimer(jobId); + getDownloadManager().cancelJob(jobId); +} diff --git a/app/src/energy-types.ts b/app/src/energy-types.ts new file mode 100644 index 0000000..a6ad27d --- /dev/null +++ b/app/src/energy-types.ts @@ -0,0 +1,92 @@ +export type EnergyRuntime = "llamacpp" | "ollama" | "vllm" | "mlx" | "transformers"; +export type EnergyMeasurement = "measured" | "estimated"; + +export interface TimeOfUseTariff { + name: string; + startHour: number; + endHour: number; + pricePerKwh: number; +} + +export interface EnergyMonitorSettings { + enabled: boolean; + electricityPricePerKwh: number; + currency: string; + timeOfUseTariffs: TimeOfUseTariff[]; + manualCpuWatts?: number; + manualGpuWatts?: number; + manualSystemIdleWatts?: number; + includeIdleSystemConsumption: boolean; + retentionDays: number; + sampleIntervalSeconds: number; + gridIntensityGCo2PerKwh?: number; +} + +export interface TariffUsageSnapshot { + name: string; + pricePerKwh: number; + currency: string; + energyKwh: number; + cost: number; +} + +export interface EnergyUsageRecord { + date: string; + runtime: EnergyRuntime; + modelId: string; + requestCount: number; + promptTokens: number; + completionTokens: number; + activeSeconds: number; + loadingSeconds: number; + energyKwh: number; + cost: number; + measurement: EnergyMeasurement; + tariffSnapshots: TariffUsageSnapshot[]; + carbonGrams: number; +} + +export interface RuntimeActivity { + id: string; + runtime: EnergyRuntime; + modelId: string; + backend: string; + device: string; + processId: number | null; + startedAt: string; + finishedAt?: string; + promptTokens: number; + completionTokens: number; + activeSeconds: number; + loadingSeconds: number; + cpuUtilization: number; + gpuUtilization: number | null; + currentPowerWatts: number; + energyKwh: number; + cost: number; + measurement: EnergyMeasurement; +} + +export interface EnergyTotals { + energyKwh: number; + cost: number; + carbonGrams: number; + requestCount: number; + promptTokens: number; + completionTokens: number; +} + +export interface EnergyDashboard { + current: RuntimeActivity[]; + today: EnergyTotals; + week: EnergyTotals; + month: EnergyTotals; + lifetime: EnergyTotals; + byModel: { key: string; totals: EnergyTotals }[]; + byRuntime: { key: string; totals: EnergyTotals }[]; + measuredPercent: number; + costPerMillionGeneratedTokens: number; + currency: string; + records: EnergyUsageRecord[]; +} + diff --git a/app/src/energy-usage-store.ts b/app/src/energy-usage-store.ts new file mode 100644 index 0000000..4b19d43 --- /dev/null +++ b/app/src/energy-usage-store.ts @@ -0,0 +1,57 @@ +import * as path from "node:path"; +import { app } from "electron"; +import { readJson, writeJson } from "./json-store"; +import type { EnergyMonitorSettings, EnergyUsageRecord, TariffUsageSnapshot } from "./energy-types"; + +function filePath(): string { + return path.join(app.getPath("userData"), "energy-usage.json"); +} + +export function listRecords(retentionDays: number): EnergyUsageRecord[] { + const records = readJson(filePath(), []); + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - Math.max(1, retentionDays)); + const cutoffDate = cutoff.toISOString().slice(0, 10); + return records.filter((record) => record.date >= cutoffDate); +} + +function mergeTariffs(existing: TariffUsageSnapshot[], incoming: TariffUsageSnapshot[]): TariffUsageSnapshot[] { + const merged = new Map(); + for (const item of [...existing, ...incoming]) { + const key = `${item.name}\u0000${item.pricePerKwh}\u0000${item.currency}`; + const current = merged.get(key); + merged.set(key, current + ? { ...current, energyKwh: current.energyKwh + item.energyKwh, cost: current.cost + item.cost } + : { ...item }); + } + return [...merged.values()]; +} + +export function addRecord(record: EnergyUsageRecord, settings: EnergyMonitorSettings): void { + const records = listRecords(settings.retentionDays); + const match = records.find((item) => + item.date === record.date + && item.runtime === record.runtime + && item.modelId === record.modelId + && item.measurement === record.measurement + ); + if (match) { + match.requestCount += record.requestCount; + match.promptTokens += record.promptTokens; + match.completionTokens += record.completionTokens; + match.activeSeconds += record.activeSeconds; + match.loadingSeconds += record.loadingSeconds; + match.energyKwh += record.energyKwh; + match.cost += record.cost; + match.carbonGrams += record.carbonGrams; + match.tariffSnapshots = mergeTariffs(match.tariffSnapshots, record.tariffSnapshots); + } else { + records.push(record); + } + writeJson(filePath(), records); +} + +export function clearRecords(): void { + writeJson(filePath(), []); +} + diff --git a/app/src/file-reader.ts b/app/src/file-reader.ts index 5d92a4d..7b3acf3 100644 --- a/app/src/file-reader.ts +++ b/app/src/file-reader.ts @@ -241,6 +241,38 @@ function walkDir(rootDir: string, dir: string, files: AttachedFile[], state: Wal } } +const MAX_PDF_FILES = 50; + +// Folder attach (openFolderAndRead/walkDir) skips .pdf entirely — it's a +// binary format the generic "dump raw content" attach flow can't use safely. +// RAG indexing wants PDFs too (for page-numbered citations), so this walks +// the same tree with the same ignored-dirs convention, collecting .pdf paths +// only, separately from the general attach path above. +export function findPdfFiles(rootDir: string): string[] { + const found: string[] = []; + function walk(dir: string): void { + if (found.length >= MAX_PDF_FILES) return; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (found.length >= MAX_PDF_FILES) return; + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name.startsWith(".") || IGNORED_DIRS.has(entry.name)) continue; + walk(fullPath); + } else if (entry.isFile() && path.extname(entry.name).toLowerCase() === ".pdf") { + found.push(fullPath); + } + } + } + walk(rootDir); + return found; +} + export async function openFolderAndRead(win: BrowserWindow | null): Promise { const result = win ? await dialog.showOpenDialog(win, { properties: ["openDirectory"] }) diff --git a/app/src/huggingface.test.ts b/app/src/huggingface.test.ts index c91d3d8..1832b20 100644 --- a/app/src/huggingface.test.ts +++ b/app/src/huggingface.test.ts @@ -1,117 +1,76 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import * as fs from "node:fs"; -import * as os from "node:os"; -import * as path from "node:path"; +import { describe, it, expect, vi, beforeEach } from "vitest"; import { downloadGgufFile } from "./huggingface"; - -function fakeResponse(opts: { status: number; headers?: Record; chunks?: string[] }) { - const chunks = (opts.chunks ?? []).map((c) => new TextEncoder().encode(c)); - let i = 0; - return { - ok: opts.status >= 200 && opts.status < 300, - status: opts.status, - headers: { get: (name: string) => opts.headers?.[name.toLowerCase()] ?? null }, - body: { - getReader: () => ({ - read: async () => (i < chunks.length ? { done: false, value: chunks[i++] } : { done: true, value: undefined }), - }), - }, - } as unknown as Response; -} - -describe("downloadGgufFile", () => { - let dir: string; - let destPath: string; - +import { downloadGgufFileNative } from "./native-downloader"; + +// The actual HTTP/resume/verify logic now lives in the Rust native addon +// (lib/), covered by its own test suite there (mock-HTTP-server-based, see +// lib/src/download/tests.rs) — a JS-side `vi.stubGlobal("fetch", ...)` can't +// reach code that no longer runs in JS. What's left to verify from the TS +// side is just this file's adapter: does it call the native function with +// the right arguments, and does it translate the native progress/error +// shapes into what callers (main.ts's hf:downloadFile handler) expect. +vi.mock("./native-downloader", () => ({ + downloadGgufFileNative: vi.fn(), +})); + +const nativeMock = vi.mocked(downloadGgufFileNative); + +describe("downloadGgufFile (native adapter)", () => { beforeEach(() => { - dir = fs.mkdtempSync(path.join(os.tmpdir(), "hf-download-test-")); - destPath = path.join(dir, "model.gguf"); - }); - - afterEach(() => { - vi.unstubAllGlobals(); + nativeMock.mockReset(); }); - it("downloads fresh, reports progress, and renames the .part file to the final name on success", async () => { - const progress: { receivedBytes: number; totalBytes: number | null }[] = []; - vi.stubGlobal( - "fetch", - vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "10" }, chunks: ["hello", "world"] })) - ); + it("calls the native addon with modelId/filename/destPath and no token when none is given", async () => { + nativeMock.mockResolvedValue(undefined); - await downloadGgufFile("org/model", "model.gguf", destPath, (p) => progress.push(p)); + await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {}); - expect(fs.existsSync(destPath)).toBe(true); - expect(fs.existsSync(destPath + ".part")).toBe(false); - expect(fs.readFileSync(destPath, "utf8")).toBe("helloworld"); - expect(progress.at(-1)).toEqual({ receivedBytes: 10, totalBytes: 10 }); + expect(nativeMock).toHaveBeenCalledWith( + "org/model", + "model.gguf", + "/tmp/model.gguf", + undefined, + expect.any(Function) + ); }); - it("resumes from an existing .part file using a Range request instead of starting over", async () => { - fs.writeFileSync(destPath + ".part", "hello"); - const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { - expect((init?.headers as Record).Range).toBe("bytes=5-"); - return fakeResponse({ status: 206, headers: { "content-range": "bytes 5-9/10" }, chunks: ["world"] }); - }); - vi.stubGlobal("fetch", fetchMock); + it("passes a null token through as undefined, matching the native addon's Option signature", async () => { + nativeMock.mockResolvedValue(undefined); - await downloadGgufFile("org/model", "model.gguf", destPath, () => {}); + await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {}, null); - expect(fs.readFileSync(destPath, "utf8")).toBe("helloworld"); + expect(nativeMock.mock.calls[0][3]).toBeUndefined(); }); - it("discards a stale partial and starts over when the server ignores the Range request", async () => { - fs.writeFileSync(destPath + ".part", "OLD-STALE-DATA"); - vi.stubGlobal( - "fetch", - vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "5" }, chunks: ["fresh"] })) - ); + it("passes a real token through unchanged", async () => { + nativeMock.mockResolvedValue(undefined); - await downloadGgufFile("org/model", "model.gguf", destPath, () => {}); + await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {}, "secret-token"); - expect(fs.readFileSync(destPath, "utf8")).toBe("fresh"); + expect(nativeMock.mock.calls[0][3]).toBe("secret-token"); }); - it("keeps the .part file (doesn't delete it) when the connection drops mid-stream, so a retry can resume", async () => { - vi.stubGlobal("fetch", vi.fn(async () => ({ - ok: true, - status: 200, - headers: { get: () => "20" }, - body: { - getReader: () => ({ - read: async () => { - throw new Error("connection reset"); - }, - }), - }, - } as unknown as Response))); - - await expect(downloadGgufFile("org/model", "model.gguf", destPath, () => {})).rejects.toThrow("connection reset"); - expect(fs.existsSync(destPath + ".part")).toBe(true); - expect(fs.existsSync(destPath)).toBe(false); - }); + it("adapts native progress events, mapping a missing totalBytes to null", async () => { + const progress: { receivedBytes: number; totalBytes: number | null }[] = []; + nativeMock.mockImplementation(async (_modelId, _filename, _dest, _token, onProgress) => { + onProgress(null, { receivedBytes: 5, totalBytes: undefined }); + onProgress(null, { receivedBytes: 10, totalBytes: 10 }); + }); - it("keeps the .part file and throws when the stream ends short of the expected size", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async () => fakeResponse({ status: 200, headers: { "content-length": "100" }, chunks: ["short"] })) - ); + await downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", (p) => progress.push(p)); - await expect(downloadGgufFile("org/model", "model.gguf", destPath, () => {})).rejects.toThrow(/incomplete/); - expect(fs.existsSync(destPath + ".part")).toBe(true); + expect(progress).toEqual([ + { receivedBytes: 5, totalBytes: null }, + { receivedBytes: 10, totalBytes: 10 }, + ]); }); - it("discards a partial and retries once when the server responds 416 (range no longer satisfiable)", async () => { - fs.writeFileSync(destPath + ".part", "stale"); - const fetchMock = vi - .fn() - .mockResolvedValueOnce(fakeResponse({ status: 416 })) - .mockResolvedValueOnce(fakeResponse({ status: 200, headers: { "content-length": "5" }, chunks: ["fresh"] })); - vi.stubGlobal("fetch", fetchMock); - - await downloadGgufFile("org/model", "model.gguf", destPath, () => {}); + it("propagates the native call's rejection unchanged, so callers' error logging keeps working", async () => { + const nativeError = new Error( + 'Download of "model.gguf" was incomplete (got 5 of 10 bytes) — try downloading it again to resume.' + ); + nativeMock.mockRejectedValue(nativeError); - expect(fs.readFileSync(destPath, "utf8")).toBe("fresh"); - expect(fetchMock).toHaveBeenCalledTimes(2); + await expect(downloadGgufFile("org/model", "model.gguf", "/tmp/model.gguf", () => {})).rejects.toBe(nativeError); }); }); diff --git a/app/src/huggingface.ts b/app/src/huggingface.ts index 3ae91a0..114c06d 100644 --- a/app/src/huggingface.ts +++ b/app/src/huggingface.ts @@ -47,20 +47,12 @@ export interface DownloadProgress { totalBytes: number | null; } -// Suffix for in-progress downloads. Writing directly to the final .gguf name -// would let a truncated file — from a network drop, a crash, or the app -// being force-quit mid-download, none of which run our error-path cleanup — -// sit there indistinguishable from a real model, so listModels() would offer -// it and loading it would fail with a confusing "corrupt GGUF" error instead -// of the app just not knowing about it. -export const PARTIAL_DOWNLOAD_SUFFIX = ".part"; - -function parseContentRangeTotal(headerValue: string | null): number | null { - // "bytes 12345-67890/98765" — the part after the slash is the full size. - const match = headerValue?.match(/\/(\d+)$/); - return match ? Number(match[1]) : null; -} - +// The actual HTTP + resume/verify logic lives in the native Rust addon +// (lib/) for speed — a plain single-stream port wouldn't outperform this, +// but splitting large files across several parallel Range-request +// connections meaningfully does. This wrapper keeps the exact signature the +// rest of the app (main.ts's hf:downloadFile handler) already calls, so +// nothing downstream needed to change. export async function downloadGgufFile( modelId: string, filename: string, @@ -68,77 +60,9 @@ export async function downloadGgufFile( onProgress: (progress: DownloadProgress) => void, token?: string | null ): Promise { - const fs = await import("node:fs"); - const url = `https://huggingface.co/${modelId}/resolve/main/${encodeURIComponent(filename)}`; - const partPath = destPath + PARTIAL_DOWNLOAD_SUFFIX; - - // A .part file left over from a dropped connection or a force-quit is - // real progress, not garbage — resume it with a Range request instead of - // re-downloading from byte zero every time. - let existingBytes = fs.existsSync(partPath) ? fs.statSync(partPath).size : 0; - const headers: Record = {}; - if (token) headers.Authorization = `Bearer ${token}`; - if (existingBytes > 0) headers.Range = `bytes=${existingBytes}-`; - - let res: Response; - try { - res = await fetch(url, { headers }); - } catch (err) { - throw new Error(`Couldn't reach Hugging Face: ${(err as Error).message}`); - } - - if (existingBytes > 0 && res.status === 416) { - // Our partial is already >= what the server has now (stale, or the - // remote file changed) — it can't be resumed, so start clean once. - fs.rmSync(partPath, { force: true }); - return downloadGgufFile(modelId, filename, destPath, onProgress, token); - } - let resuming = existingBytes > 0 && res.status === 206; - if (existingBytes > 0 && !resuming) { - // Server ignored the Range request and is sending the whole file - // from the start — appending that to the stale partial would - // corrupt it, so discard the partial and treat this as fresh. - existingBytes = 0; - } - - if (!res.ok || !res.body) throw new Error(`Failed to download "${filename}" (HTTP ${res.status}).`); - - const contentLength = Number(res.headers.get("content-length")) || null; - const totalBytes = resuming - ? (parseContentRangeTotal(res.headers.get("content-range")) ?? (contentLength !== null ? existingBytes + contentLength : null)) - : contentLength; - - let receivedBytes = existingBytes; - const writeStream = fs.createWriteStream(partPath, { flags: resuming ? "a" : "w" }); - const reader = res.body.getReader(); - onProgress({ receivedBytes, totalBytes }); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - receivedBytes += value.byteLength; - onProgress({ receivedBytes, totalBytes }); - await new Promise((resolve, reject) => { - writeStream.write(value, (err) => (err ? reject(err) : resolve())); - }); - } - } catch (err) { - // Deliberately not deleting partPath — a dropped connection here - // leaves real, resumable progress on disk for the next attempt. - // Waited out rather than fire-and-forget so the partial bytes are - // actually flushed to disk before this function returns. - await new Promise((resolve) => writeStream.end(() => resolve())); - throw err; - } - await new Promise((resolve, reject) => { - writeStream.once("error", reject); - writeStream.end(() => resolve()); + const { downloadGgufFileNative } = await import("./native-downloader"); + await downloadGgufFileNative(modelId, filename, destPath, token ?? undefined, (err, progress) => { + if (err) return; // fatal errors surface via the returned promise's rejection instead + onProgress({ receivedBytes: progress.receivedBytes, totalBytes: progress.totalBytes ?? null }); }); - if (totalBytes !== null && receivedBytes !== totalBytes) { - throw new Error( - `Download of "${filename}" was incomplete (got ${receivedBytes} of ${totalBytes} bytes) — try downloading it again to resume.` - ); - } - fs.renameSync(partPath, destPath); } diff --git a/app/src/json-store.test.ts b/app/src/json-store.test.ts index 114634d..62e2259 100644 --- a/app/src/json-store.test.ts +++ b/app/src/json-store.test.ts @@ -34,6 +34,34 @@ describe("json-store", () => { expect(entries).toEqual(["data.json"]); }); + // These files hold API keys and conversation history; the default umask + // would leave them readable by other accounts on the machine. + it.skipIf(process.platform === "win32")("writes owner-only files", () => { + writeJson(file, { token: "value" }); + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + }); + + it.skipIf(process.platform === "win32")("keeps the file owner-only on rewrite", () => { + writeJson(file, { token: "first" }); + writeJson(file, { token: "second" }); + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + }); + + it.skipIf(process.platform === "win32")("stays owner-only when a stale temp file exists", () => { + const stale = `${file}.tmp-${process.pid}`; + fs.writeFileSync(stale, "leftover", { mode: 0o666 }); + writeJson(file, { token: "value" }); + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + }); + + // A file written by an older build is only ever read if its contents never + // change, so tightening on write alone would never reach it. + it.skipIf(process.platform === "win32")("tightens an existing world-readable file on read", () => { + fs.writeFileSync(file, JSON.stringify({ token: "value" }), { mode: 0o644 }); + expect(readJson(file, {})).toEqual({ token: "value" }); + expect(fs.statSync(file).mode & 0o777).toBe(0o600); + }); + it("backs up and falls back to the default when the file is corrupted", () => { fs.writeFileSync(file, "{ not valid json"); const result = readJson(file, { safe: true }); diff --git a/app/src/json-store.ts b/app/src/json-store.ts index 08732fb..99d2378 100644 --- a/app/src/json-store.ts +++ b/app/src/json-store.ts @@ -30,6 +30,8 @@ export function readJson(filePath: string, fallback: T): T { return fallback; } + restrictExistingPermissions(filePath); + try { return JSON.parse(raw) as T; } catch (err) { @@ -43,9 +45,48 @@ export function readJson(filePath: string, fallback: T): T { } } +// These files hold provider API keys (secrets.json) and full conversation +// history. Written with the process umask they land as 0644 — or 0664 under +// the umask 002 several distributions ship — leaving their contents readable +// to anything that reaches them: another account on a shared machine, a +// backup or sync tool, an archive unpacked somewhere else. The userData +// directory is usually restrictive enough to cover that on a single-user +// desktop, but a stored credential shouldn't depend on its parent directory's +// mode. +const PRIVATE_FILE_MODE = 0o600; + +// Files written before this existed keep the mode they were created with, and +// writeJson alone would never reach them: a key set once and never changed is +// only ever read. So the mode is also tightened on first read, once per path +// per run to keep it off the hot path. +const permissionsChecked = new Set(); + +function restrictExistingPermissions(filePath: string): void { + if (process.platform === "win32" || permissionsChecked.has(filePath)) return; + permissionsChecked.add(filePath); + try { + if ((fs.statSync(filePath).mode & 0o777) !== PRIVATE_FILE_MODE) { + fs.chmodSync(filePath, PRIVATE_FILE_MODE); + } + } catch (err) { + // Best effort — unusual ownership or an exotic filesystem must not + // stop the app from reading its own data. + logger.error(`Failed to restrict permissions on ${filePath}: ${(err as Error).message}`); + } +} + export function writeJson(filePath: string, data: unknown): void { fs.mkdirSync(path.dirname(filePath), { recursive: true }); const tmpPath = `${filePath}.tmp-${process.pid}`; - fs.writeFileSync(tmpPath, JSON.stringify(data, null, 2)); + // The mode goes on the temp file, because the rename below replaces the + // destination inode and takes the temp file's mode with it. Chmod'ing the + // destination afterwards would leave a window where the contents are + // readable, and would be undone by the next write. + // + // Removed first so writeFileSync always creates the file and so always + // applies `mode` — it ignores the option for a path that already exists, + // and an interrupted earlier write can leave one behind under this pid. + fs.rmSync(tmpPath, { force: true }); + fs.writeFileSync(tmpPath, JSON.stringify(data, null, 2), { mode: PRIVATE_FILE_MODE }); fs.renameSync(tmpPath, filePath); } diff --git a/app/src/local-server-manager.test.ts b/app/src/local-server-manager.test.ts index 427f349..98e5943 100644 --- a/app/src/local-server-manager.test.ts +++ b/app/src/local-server-manager.test.ts @@ -1,9 +1,9 @@ import { describe, it, expect } from "vitest"; -import { buildRuntimeProbe, buildServerCommand, describeSpawnFailure } from "./local-server-manager"; +import { buildRuntimeProbe, buildServerCommand, describeSpawnFailure, identityMatches, explainStartupFailure } from "./local-server-manager"; describe("buildServerCommand", () => { it("builds an mlx_lm.server invocation with the default python", () => { - const { command, args } = buildServerCommand("mlx", "mlx-community/Llama-3.2-3B-Instruct-4bit", {}); + const { command, args } = buildServerCommand("mlx", "mlx-community/Llama-3.2-3B-Instruct-4bit", {}, "darwin", 49151); expect(command).toBe("python3"); expect(args).toContain("mlx_lm.server"); expect(args).toContain("mlx-community/Llama-3.2-3B-Instruct-4bit"); @@ -12,14 +12,14 @@ describe("buildServerCommand", () => { }); it("respects a custom python interpreter path", () => { - const { command } = buildServerCommand("mlx", "some/model", { mlxPythonPath: "/opt/python/bin/python3.12" }); + const { command } = buildServerCommand("mlx", "some/model", { mlxPythonPath: "/opt/python/bin/python3.12" }, "darwin", 49151); expect(command).toBe("/opt/python/bin/python3.12"); }); it("builds a llama-server invocation with full GPU offload for rocm", () => { const { command, args } = buildServerCommand("rocm", "/models/llama.gguf", { rocmServerPath: "/opt/rocm-llama/llama-server", - }); + }, "linux", 49152); expect(command).toBe("/opt/rocm-llama/llama-server"); expect(args).toEqual( expect.arrayContaining(["-m", "/models/llama.gguf", "--n-gpu-layers", "999"]) @@ -27,21 +27,21 @@ describe("buildServerCommand", () => { }); it("falls back to PATH lookup when no rocm binary path is configured", () => { - const { command } = buildServerCommand("rocm", "/models/llama.gguf", {}); + const { command } = buildServerCommand("rocm", "/models/llama.gguf", {}, "linux", 49152); expect(command).toBe("llama-server"); }); - it("uses distinct fixed ports per backend so both can run at once", () => { - const mlx = buildServerCommand("mlx", "m", {}); - const rocm = buildServerCommand("rocm", "m", {}); - const vllm = buildServerCommand("vllm", "m", {}); + it("uses the dynamically allocated port supplied to each backend", () => { + const mlx = buildServerCommand("mlx", "m", {}, "darwin", 49151); + const rocm = buildServerCommand("rocm", "m", {}, "linux", 49152); + const vllm = buildServerCommand("vllm", "m", {}, "linux", 49153); const portOf = (args: string[]) => args[args.indexOf("--port") + 1]; expect(portOf(mlx.args)).not.toBe(portOf(rocm.args)); expect(new Set([portOf(mlx.args), portOf(rocm.args), portOf(vllm.args)]).size).toBe(3); }); it("builds a managed vLLM OpenAI server command", () => { - const { command, args } = buildServerCommand("vllm", "meta-llama/Llama-3.1-8B-Instruct", {}, "linux"); + const { command, args } = buildServerCommand("vllm", "meta-llama/Llama-3.1-8B-Instruct", {}, "linux", 49153); expect(command).toBe("vllm"); expect(args).toEqual( expect.arrayContaining(["serve", "meta-llama/Llama-3.1-8B-Instruct", "--host", "127.0.0.1"]) @@ -49,17 +49,30 @@ describe("buildServerCommand", () => { }); it("allows a vLLM command override without requiring one", () => { - const { command } = buildServerCommand("vllm", "some/model", { vllmCommand: "/opt/vllm/bin/vllm" }); + const { command } = buildServerCommand("vllm", "some/model", { vllmCommand: "/opt/vllm/bin/vllm" }, "linux", 49153); expect(command).toBe("/opt/vllm/bin/vllm"); }); it("launches vLLM through WSL automatically on Windows", () => { - const { command, args } = buildServerCommand("vllm", "some/model", {}, "win32"); + const { command, args } = buildServerCommand("vllm", "some/model", {}, "win32", 49153); expect(command).toBe("wsl.exe"); expect(args.slice(0, 4)).toEqual(["--", "vllm", "serve", "some/model"]); }); }); +describe("runtime identity health", () => { + it("requires an OpenAI model list containing the expected model", () => { + expect(identityMatches({ data: [{ id: "meta-llama/Llama-3.1-8B-Instruct" }] }, "meta-llama/Llama-3.1-8B-Instruct")).toBe(true); + expect(identityMatches({ status: "ok" }, "meta-llama/Llama-3.1-8B-Instruct")).toBe(false); + expect(identityMatches({ data: [{ id: "another-model" }] }, "meta-llama/Llama-3.1-8B-Instruct")).toBe(false); + }); + + it("turns common startup logs into actionable explanations", () => { + expect(explainStartupFailure("vllm", ["CUDA out of memory"])).toMatch(/RAM or VRAM/); + expect(explainStartupFailure("mlx", ["ModuleNotFoundError: No module named mlx_lm"])).toMatch(/pip install mlx-lm/); + }); +}); + describe("describeSpawnFailure", () => { it("points mlx failures at the mlx-lm install", () => { expect(describeSpawnFailure("mlx")).toMatch(/mlx-lm/); diff --git a/app/src/local-server-manager.ts b/app/src/local-server-manager.ts index 9d0e2bd..dcf1038 100644 --- a/app/src/local-server-manager.ts +++ b/app/src/local-server-manager.ts @@ -1,374 +1,225 @@ -import { spawn, ChildProcess } from "node:child_process"; +import { spawn, execFile, type ChildProcess } from "node:child_process"; +import { createServer } from "node:net"; +import { promisify } from "node:util"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import pidusage from "pidusage"; import { logger } from "./logger"; +import { killProcessTree } from "./process-tree"; +import { environmentPython } from "./python-runtime-manager"; -// Inference backends node-llama-cpp can't serve (it only ships CUDA/Vulkan/ -// Metal prebuilds) are run as external server processes that expose the -// OpenAI-compatible chat-completions API, and chat traffic goes through the -// same openai-compatible client the cloud providers use: -// -// - "mlx": Apple-Silicon inference via Python's mlx_lm package -// (`python3 -m mlx_lm.server`). Models are Hugging Face repo ids or local -// paths; the server downloads/loads them itself. -// - "rocm": AMD-GPU inference via a ROCm/HIP build of llama.cpp's -// `llama-server` binary (the official llama.cpp releases ship one), run -// against the same GGUF files the built-in llama.cpp backend uses. +const execFileAsync = promisify(execFile); export type LocalBackendId = "mlx" | "rocm" | "vllm"; +export type RuntimeLifecycleState = "starting" | "running" | "unhealthy" | "stopped"; -export interface LocalBackendConfig { - // Path to the ROCm llama-server binary. No sensible default beyond PATH - // lookup — the user downloads a HIP build themselves. - rocmServerPath?: string; - // Python interpreter used to launch mlx_lm.server (needs `pip install mlx-lm`). - mlxPythonPath?: string; - // Optional override; managed vLLM uses the `vllm` command from PATH. - vllmCommand?: string; -} - +export interface LocalBackendConfig { rocmServerPath?: string; mlxPythonPath?: string; vllmCommand?: string } +export interface RuntimeProbe { compatible: boolean; command: string; args: string[]; detail: string } export interface LocalRuntimeStatus { - backend: LocalBackendId; - compatible: boolean; - installed: boolean; - running: boolean; - model?: string; - detail: string; -} - -export interface RuntimeProbe { - compatible: boolean; - command: string; - args: string[]; - detail: string; + backend: LocalBackendId; compatible: boolean; installed: boolean; running: boolean; state: RuntimeLifecycleState; + model?: string; detail: string; device?: string; pid: number | null; port: number | null; startedAt: string | null; + uptimeSeconds: number; ramMB: number | null; vramMB: number | null; logs: string[]; startupError?: string; + installCommand: string; environmentIssues: string[]; } interface RunningServer { - process: ChildProcess; - model: string; - baseUrl: string; - exited: boolean; - activeRequests: number; - idleTimer: NodeJS.Timeout | null; + process: ChildProcess; model: string; baseUrl: string; port: number; state: RuntimeLifecycleState; exited: boolean; + startedAt: number; activeRequests: number; idleTimer: NodeJS.Timeout | null; logs: string[]; logRemainder: string; + startupError?: string; } -// Fixed per-backend ports so a restarted app reconnects rather than leaking -// orphan servers across random ports. -const PORTS: Record = { mlx: 8790, rocm: 8791, vllm: 8792 }; -// First startup can include downloading/loading a multi-GB model. const STARTUP_TIMEOUT_MS = 180_000; const HEALTH_POLL_MS = 750; +const MAX_LOG_LINES = 500; +const MAX_LOG_LINE_CHARS = 4_000; const configuredIdleMinutes = Number(process.env.OLLAMA_CUSTOM_UI_LOCAL_BACKEND_IDLE_MINUTES ?? 10); -const IDLE_TIMEOUT_MS = Number.isFinite(configuredIdleMinutes) - ? Math.max(0, configuredIdleMinutes) * 60_000 - : 10 * 60_000; - +const IDLE_TIMEOUT_MS = Number.isFinite(configuredIdleMinutes) ? Math.max(0, configuredIdleMinutes) * 60_000 : 600_000; const servers = new Map(); const serverStarts = new Map }>(); +const stoppedSnapshots = new Map>(); +const probeCache = new Map(); -export function buildRuntimeProbe( - backend: LocalBackendId, - config: LocalBackendConfig, - platform: NodeJS.Platform = process.platform, - arch: string = process.arch -): RuntimeProbe { +export function buildRuntimeProbe(backend: LocalBackendId, config: LocalBackendConfig, platform: NodeJS.Platform = process.platform, arch = process.arch): RuntimeProbe { if (backend === "mlx") { const compatible = platform === "darwin" && arch === "arm64"; - return { - compatible, - command: config.mlxPythonPath?.trim() || "python3", - args: ["-c", "import mlx_lm"], - detail: compatible ? "Apple Silicon accelerated runtime" : "Requires an Apple Silicon Mac", - }; + const managedPython = environmentPython("mlx", platform); + return { compatible, command: config.mlxPythonPath?.trim() || (fs.existsSync(managedPython) ? managedPython : "python3"), args: ["-c", "import mlx_lm"], detail: compatible ? "Apple Silicon accelerated runtime" : "Requires an Apple Silicon Mac" }; } if (backend === "vllm") { const compatible = platform === "linux" || platform === "win32"; - if (!config.vllmCommand?.trim() && platform === "win32") { - return { - compatible, - command: "wsl.exe", - args: ["--", "vllm", "--version"], - detail: "CUDA or ROCm runtime through WSL", - }; - } - return { - compatible, - command: config.vllmCommand?.trim() || "vllm", - args: ["--version"], - detail: compatible ? "High-throughput CUDA or ROCm runtime" : "Requires Linux or Windows with WSL", - }; + if (!config.vllmCommand?.trim() && platform === "win32") return { compatible, command: "wsl.exe", args: ["--", "vllm", "--version"], detail: "CUDA or ROCm runtime through WSL" }; + return { compatible, command: config.vllmCommand?.trim() || "vllm", args: ["--version"], detail: compatible ? "High-throughput CUDA or ROCm runtime" : "Requires Linux or Windows with WSL" }; } const compatible = platform === "linux" || !!config.rocmServerPath?.trim(); - return { - compatible, - command: config.rocmServerPath?.trim() || "llama-server", - args: ["--version"], - detail: compatible ? "AMD GPU runtime for local GGUF models" : "Requires Linux and a ROCm-capable AMD GPU", - }; + return { compatible, command: config.rocmServerPath?.trim() || "llama-server", args: ["--version"], detail: compatible ? "AMD GPU runtime for local GGUF models" : "Requires Linux and a ROCm-capable AMD GPU" }; +} + +export function installCommand(backend: LocalBackendId): string { + if (backend === "mlx") return "python3 -m pip install mlx-lm"; + if (backend === "vllm") return "python3 -m pip install vllm"; + return "Install a ROCm/HIP llama-server build, then configure its executable in Settings"; +} + +export function describeSpawnFailure(backend: LocalBackendId): string { + if (backend === "mlx") return `Couldn't launch MLX. Install mlx-lm on an Apple Silicon Mac (${installCommand("mlx")}) and verify the configured Python interpreter.`; + if (backend === "vllm") return `Couldn't launch vLLM. Install it in Linux/WSL (${installCommand("vllm")}) and verify that its CUDA or ROCm requirements match the GPU driver.`; + return `Couldn't launch ROCm llama-server. ${installCommand("rocm")} and verify ROCm device access.`; } async function commandSucceeds(command: string, args: string[]): Promise { return new Promise((resolve) => { - let settled = false; - let timer: NodeJS.Timeout; - const finish = (value: boolean) => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(value); - }; - let child: ChildProcess; - try { - child = spawn(command, args, { stdio: "ignore" }); - } catch { - resolve(false); - return; - } - timer = setTimeout(() => { - child.kill(); - finish(false); - }, 5_000); - timer.unref(); - child.once("error", () => finish(false)); - child.once("exit", (code) => finish(code === 0)); + let settled = false; let child: ChildProcess; + const finish = (value: boolean) => { if (settled) return; settled = true; clearTimeout(timer); resolve(value); }; + try { child = spawn(command, args, { stdio: "ignore", windowsHide: true }); } catch { resolve(false); return; } + const timer = setTimeout(() => { child.kill(); finish(false); }, 5_000); timer.unref(); + child.once("error", () => finish(false)); child.once("exit", (code) => finish(code === 0)); }); } -export async function getRuntimeStatuses(config: LocalBackendConfig): Promise { - return Promise.all( - (["rocm", "mlx", "vllm"] as const).map(async (backend) => { - const probe = buildRuntimeProbe(backend, config); - const running = servers.get(backend); - // A live, non-exited process proves the runtime is installed - // without needing to probe. Anything else — no entry, or a - // stale one left behind after the process crashed — has to - // fall through to an actual probe; treating "exited" the same - // as "confirmed not installed" (the previous `running ? ... :` - // shortcut did, since !running.exited is false either way) - // reported a crashed-but-installed runtime as absent. - const installed = probe.compatible && (running && !running.exited ? true : await commandSucceeds(probe.command, probe.args)); - return { - backend, - compatible: probe.compatible, - installed, - running: !!running && !running.exited, - model: running && !running.exited ? running.model : undefined, - detail: probe.detail, - }; - }) - ); +async function cachedCommandSucceeds(command: string, args: string[]): Promise { + const key = JSON.stringify([command, args]); const cached = probeCache.get(key); + if (cached && cached.expiresAt > Date.now()) return cached.installed; + const installed = await commandSucceeds(command, args); + probeCache.set(key, { installed, expiresAt: Date.now() + 30_000 }); + return installed; } -function clearIdleTimer(server: RunningServer): void { - if (!server.idleTimer) return; - clearTimeout(server.idleTimer); - server.idleTimer = null; -} - -function scheduleIdleStop(backend: LocalBackendId, server: RunningServer): void { - clearIdleTimer(server); - if (IDLE_TIMEOUT_MS === 0 || server.activeRequests > 0 || server.exited) return; - server.idleTimer = setTimeout(() => { - server.idleTimer = null; - if (servers.get(backend) === server && server.activeRequests === 0) { - logger.info(`Stopping idle ${backend} runtime to release GPU memory`); - stopServer(backend); - } - }, IDLE_TIMEOUT_MS); - server.idleTimer.unref(); +export async function allocatePort(): Promise { + return new Promise((resolve, reject) => { + const socket = createServer(); + socket.unref(); socket.once("error", reject); + socket.listen({ host: "127.0.0.1", port: 0, exclusive: true }, () => { + const address = socket.address(); + if (!address || typeof address === "string") { socket.close(); reject(new Error("Could not allocate a local runtime port")); return; } + const port = address.port; socket.close((error) => error ? reject(error) : resolve(port)); + }); + }); } -export function buildServerCommand( - backend: LocalBackendId, - model: string, - config: LocalBackendConfig, - platform: NodeJS.Platform = process.platform -): { command: string; args: string[] } { - const port = PORTS[backend]; - if (backend === "mlx") { - return { - command: config.mlxPythonPath?.trim() || "python3", - args: ["-m", "mlx_lm.server", "--model", model, "--port", String(port), "--host", "127.0.0.1"], - }; - } +export function buildServerCommand(backend: LocalBackendId, model: string, config: LocalBackendConfig, platform: NodeJS.Platform = process.platform, port = 0): { command: string; args: string[] } { + if (!port) throw new Error("A dynamically allocated runtime port is required"); + if (backend === "mlx") { const managed = environmentPython("mlx", platform); return { command: config.mlxPythonPath?.trim() || (fs.existsSync(managed) ? managed : "python3"), args: ["-m", "mlx_lm.server", "--model", model, "--port", String(port), "--host", "127.0.0.1"] }; } if (backend === "vllm") { const args = ["serve", model, "--port", String(port), "--host", "127.0.0.1"]; - if (!config.vllmCommand?.trim() && platform === "win32") { - return { command: "wsl.exe", args: ["--", "vllm", ...args] }; - } - return { - command: config.vllmCommand?.trim() || "vllm", - args, - }; + const cudaExecutable = path.join(path.dirname(environmentPython("vllm-cuda", platform)), platform === "win32" ? "vllm.exe" : "vllm"); + const rocmExecutable = path.join(path.dirname(environmentPython("vllm-rocm", platform)), platform === "win32" ? "vllm.exe" : "vllm"); + const managed = fs.existsSync(cudaExecutable) ? cudaExecutable : fs.existsSync(rocmExecutable) ? rocmExecutable : undefined; + return !config.vllmCommand?.trim() && !managed && platform === "win32" ? { command: "wsl.exe", args: ["--", "vllm", ...args] } : { command: config.vllmCommand?.trim() || managed || "vllm", args }; } - return { - command: config.rocmServerPath?.trim() || "llama-server", - args: [ - "-m", model, - "--port", String(port), - "--host", "127.0.0.1", - // Offload everything; llama-server clamps to what actually fits. - "--n-gpu-layers", "999", - ], - }; + return { command: config.rocmServerPath?.trim() || "llama-server", args: ["-m", model, "--port", String(port), "--host", "127.0.0.1", "--n-gpu-layers", "999"] }; } -export function describeSpawnFailure(backend: LocalBackendId): string { - if (backend === "mlx") { - return "Couldn't launch the managed MLX runtime — install mlx-lm (pip install mlx-lm) on an Apple Silicon Mac."; - } - if (backend === "vllm") { - return "Couldn't launch the managed vLLM runtime — install vLLM so the vllm command is available (pip install vllm)."; - } - return "Couldn't launch the managed ROCm runtime — install a ROCm/HIP llama-server build and make llama-server available on PATH."; +function pushLog(entry: RunningServer, source: "stdout" | "stderr" | "manager", text: string): void { + const combined = entry.logRemainder + text; const lines = combined.split(/\r?\n/); entry.logRemainder = lines.pop() ?? ""; + for (const line of lines) { if (!line) continue; entry.logs.push(`${new Date().toISOString()} [${source}] ${line.slice(0, MAX_LOG_LINE_CHARS)}`); } + if (entry.logs.length > MAX_LOG_LINES) entry.logs.splice(0, entry.logs.length - MAX_LOG_LINES); +} + +export function identityMatches(payload: unknown, expectedModel: string): boolean { + if (!payload || typeof payload !== "object") return false; + const data = (payload as { data?: unknown }).data; + if (!Array.isArray(data) || data.length === 0) return false; + const ids = data.map((item) => String((item as { id?: unknown })?.id ?? "").toLowerCase()).filter(Boolean); + if (!ids.length) return false; + const expected = expectedModel.toLowerCase(); const leaf = expected.split(/[\\/]/).pop() ?? expected; + return ids.some((id) => id === expected || id.includes(leaf) || expected.includes(id)); } -// Any HTTP response means the server socket is up (a 404 from a route probe -// is still proof of life); only a network-level failure counts as down. -async function isReachable(baseUrl: string): Promise { +async function healthCheck(baseUrl: string, expectedModel: string): Promise { try { - await fetch(`${baseUrl}/v1/models`, { signal: AbortSignal.timeout(2000) }); - return true; - } catch { - return false; - } + const response = await fetch(`${baseUrl}/v1/models`, { signal: AbortSignal.timeout(2_000) }); + if (!response.ok || !(response.headers.get("content-type") ?? "").includes("json")) return false; + return identityMatches(await response.json(), expectedModel); + } catch { return false; } } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); +function clearIdleTimer(server: RunningServer): void { if (server.idleTimer) clearTimeout(server.idleTimer); server.idleTimer = null; } +function scheduleIdleStop(backend: LocalBackendId, server: RunningServer): void { + clearIdleTimer(server); if (!IDLE_TIMEOUT_MS || server.activeRequests || server.exited) return; + server.idleTimer = setTimeout(() => { if (servers.get(backend) === server && !server.activeRequests) stopServer(backend); }, IDLE_TIMEOUT_MS); server.idleTimer.unref(); } -async function startOrReuseServer( - backend: LocalBackendId, - model: string, - config: LocalBackendConfig -): Promise { +async function startOrReuseServer(backend: LocalBackendId, model: string, config: LocalBackendConfig): Promise { const existing = servers.get(backend); - if (existing && !existing.exited && existing.model === model) { - if (await isReachable(existing.baseUrl)) return existing.baseUrl; - // Process alive but unresponsive — restart it below. - } - if (existing) { - if (existing.activeRequests > 0) { - throw new Error(`The ${backend} runtime is busy. Wait for the active response before changing models.`); - } - clearIdleTimer(existing); - existing.process.kill(); - servers.delete(backend); - } - - const baseUrl = `http://127.0.0.1:${PORTS[backend]}`; - const { command, args } = buildServerCommand(backend, model, config); - logger.info(`Starting ${backend} server: ${command} ${args.join(" ")}`); - + if (existing && !existing.exited && existing.model === model && await healthCheck(existing.baseUrl, model)) return existing.baseUrl; + if (existing) { if (existing.activeRequests) throw new Error(`The ${backend} runtime is busy.`); stopServer(backend); } + const port = await allocatePort(); const baseUrl = `http://127.0.0.1:${port}`; const { command, args } = buildServerCommand(backend, model, config, process.platform, port); let child: ChildProcess; - try { - child = spawn(command, args, { stdio: "ignore" }); - } catch { - throw new Error(describeSpawnFailure(backend)); - } - - const entry: RunningServer = { - process: child, - model, - baseUrl, - exited: false, - activeRequests: 0, - idleTimer: null, - }; - servers.set(backend, entry); - + try { child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, detached: process.platform !== "win32" }); } catch { throw new Error(describeSpawnFailure(backend)); } + const entry: RunningServer = { process: child, model, baseUrl, port, state: "starting", exited: false, startedAt: Date.now(), activeRequests: 0, idleTimer: null, logs: [], logRemainder: "" }; + servers.set(backend, entry); pushLog(entry, "manager", `Starting ${command} ${args.join(" ")}`); + child.stdout?.setEncoding("utf8"); child.stderr?.setEncoding("utf8"); child.stdout?.on("data", (data: string) => pushLog(entry, "stdout", data)); child.stderr?.on("data", (data: string) => pushLog(entry, "stderr", data)); let spawnError: string | null = null; - child.on("error", (err) => { - spawnError = (err as NodeJS.ErrnoException).code === "ENOENT" ? describeSpawnFailure(backend) : err.message; - entry.exited = true; - }); - child.on("exit", (code) => { - entry.exited = true; - if (code !== 0 && code !== null) logger.warn(`${backend} server exited with code ${code}`); - }); - + child.on("error", (error: NodeJS.ErrnoException) => { spawnError = error.code === "ENOENT" ? describeSpawnFailure(backend) : error.message; entry.startupError = spawnError; entry.state = "unhealthy"; entry.exited = true; }); + child.on("exit", (code, signal) => { entry.exited = true; entry.state = code === 0 ? "stopped" : "unhealthy"; pushLog(entry, "manager", `Process exited code=${code ?? "null"} signal=${signal ?? "none"}`); if (code) entry.startupError = explainStartupFailure(backend, entry.logs); }); const deadline = Date.now() + STARTUP_TIMEOUT_MS; while (Date.now() < deadline) { if (spawnError) throw new Error(spawnError); - if (entry.exited) { - servers.delete(backend); - throw new Error( - backend === "mlx" - ? "The MLX runtime exited during startup — check that mlx-lm is installed and the model id is valid." - : backend === "vllm" - ? "The vLLM runtime exited during startup — check that vLLM supports this model and that enough GPU memory is available." - : "The ROCm runtime exited during startup — check that llama-server is a working HIP build and the model is a valid GGUF." - ); - } - if (await isReachable(baseUrl)) return baseUrl; - await sleep(HEALTH_POLL_MS); + if (entry.exited) { const explanation = entry.startupError ?? explainStartupFailure(backend, entry.logs); stoppedSnapshots.set(backend, { logs: [...entry.logs], startupError: explanation, model }); servers.delete(backend); throw new Error(explanation); } + if (await healthCheck(baseUrl, model)) { entry.state = "running"; pushLog(entry, "manager", `Identity health check passed on port ${port}`); scheduleIdleStop(backend, entry); return baseUrl; } + await new Promise((resolve) => setTimeout(resolve, HEALTH_POLL_MS)); } - child.kill(); - servers.delete(backend); - throw new Error(`The ${backend} server didn't become reachable within ${STARTUP_TIMEOUT_MS / 1000}s.`); + entry.state = "unhealthy"; entry.startupError = `Runtime did not identify itself within ${STARTUP_TIMEOUT_MS / 1000}s. ${explainStartupFailure(backend, entry.logs)}`; stopServer(backend); throw new Error(entry.startupError); } -export async function ensureServer( - backend: LocalBackendId, - model: string, - config: LocalBackendConfig -): Promise { - const pending = serverStarts.get(backend); - if (pending) { - if (pending.model === model) return pending.promise; - await pending.promise.catch(() => undefined); - return ensureServer(backend, model, config); - } +export function explainStartupFailure(backend: LocalBackendId, logs: string[]): string { + const text = logs.slice(-80).join("\n"); + if (/out of memory|cuda.*memory|hip.*memory|cannot allocate/i.test(text)) return `${backend} could not load the model because available RAM or VRAM is insufficient.`; + if (/no module named|module.*not found/i.test(text)) return `${backend} is missing a Python package. ${installCommand(backend)}.`; + if (/permission denied|eacces/i.test(text)) return `${backend} could not execute or access the model because of filesystem permissions.`; + if (/not found|enoent|no such file/i.test(text)) return `${backend} could not find its executable or model. Validate the configured paths.`; + if (/cuda|driver|rocm|hip/i.test(text)) return `${backend} failed GPU initialization. Verify the driver, runtime version, and device compatibility.`; + return `${backend} exited before becoming healthy. Review the captured runtime logs for the original error.`; +} - const promise = startOrReuseServer(backend, model, config); - serverStarts.set(backend, { model, promise }); - try { - return await promise; - } finally { - if (serverStarts.get(backend)?.promise === promise) serverStarts.delete(backend); - } +export async function ensureServer(backend: LocalBackendId, model: string, config: LocalBackendConfig): Promise { + const pending = serverStarts.get(backend); if (pending) { if (pending.model === model) return pending.promise; await pending.promise.catch(() => undefined); return ensureServer(backend, model, config); } + const promise = startOrReuseServer(backend, model, config); serverStarts.set(backend, { model, promise }); + try { return await promise; } finally { if (serverStarts.get(backend)?.promise === promise) serverStarts.delete(backend); } } +export async function startServer(backend: LocalBackendId, model: string, config: LocalBackendConfig): Promise { return ensureServer(backend, model, config); } +export async function restartServer(backend: LocalBackendId, model: string, config: LocalBackendConfig): Promise { stopServer(backend); return ensureServer(backend, model, config); } -export async function acquireServer( - backend: LocalBackendId, - model: string, - config: LocalBackendConfig -): Promise<{ baseUrl: string; release(): void }> { - const current = servers.get(backend); - if (current) clearIdleTimer(current); - const baseUrl = await ensureServer(backend, model, config); - const server = servers.get(backend); - if (!server || server.exited || server.model !== model) { - throw new Error(`The ${backend} runtime stopped before the request could start.`); - } - server.activeRequests++; - let released = false; - return { - baseUrl, - release(): void { - if (released) return; - released = true; - if (servers.get(backend) !== server) return; - server.activeRequests = Math.max(0, server.activeRequests - 1); - scheduleIdleStop(backend, server); - }, - }; +export async function acquireServer(backend: LocalBackendId, model: string, config: LocalBackendConfig): Promise<{ baseUrl: string; release(): void }> { + const current = servers.get(backend); if (current) clearIdleTimer(current); const baseUrl = await ensureServer(backend, model, config); const server = servers.get(backend); + if (!server || server.exited || server.model !== model) throw new Error(`The ${backend} runtime stopped before the request could start.`); + server.activeRequests++; let released = false; + return { baseUrl, release() { if (released) return; released = true; if (servers.get(backend) !== server) return; server.activeRequests = Math.max(0, server.activeRequests - 1); scheduleIdleStop(backend, server); } }; } export function stopServer(backend: LocalBackendId): void { - const entry = servers.get(backend); - if (entry) { - clearIdleTimer(entry); - entry.process.kill(); - servers.delete(backend); - } + const entry = servers.get(backend); if (!entry) return; clearIdleTimer(entry); pushLog(entry, "manager", "Stop requested"); + stoppedSnapshots.set(backend, { logs: [...entry.logs], startupError: entry.startupError, model: entry.model }); + const pid = entry.process.pid; entry.process.kill("SIGTERM"); + if (pid) { const timer = setTimeout(() => { if (!entry.exited) killProcessTree(pid, "SIGKILL"); }, 5_000); timer.unref(); } + servers.delete(backend); } +export function stopAll(): void { serverStarts.clear(); for (const backend of [...servers.keys()]) stopServer(backend); } +export function getRunningBackends(): { backend: LocalBackendId; model: string }[] { return [...servers].filter(([, server]) => !server.exited).map(([backend, server]) => ({ backend, model: server.model })); } -export function stopAll(): void { - serverStarts.clear(); - for (const backend of [...servers.keys()]) stopServer(backend); +async function gpuProcessMemory(pid: number): Promise<{ device?: string; vramMB: number | null }> { + try { + const { stdout } = await execFileAsync("nvidia-smi", ["--query-compute-apps=pid,gpu_name,used_memory", "--format=csv,noheader,nounits"], { timeout: 2_000, windowsHide: true }); + for (const line of stdout.split(/\r?\n/)) { const [rawPid, device, memory] = line.split(",").map((item) => item.trim()); if (Number(rawPid) === pid) return { device, vramMB: Number(memory) }; } + } catch { /* unavailable or non-NVIDIA */ } + return { vramMB: null }; } -export function getRunningBackends(): { backend: LocalBackendId; model: string }[] { - return [...servers.entries()] - .filter(([, s]) => !s.exited) - .map(([backend, s]) => ({ backend, model: s.model })); +export async function getRuntimeStatuses(config: LocalBackendConfig): Promise { + return Promise.all((["rocm", "mlx", "vllm"] as const).map(async (backend) => { + const probe = buildRuntimeProbe(backend, config); const server = servers.get(backend); const live = !!server && !server.exited; + const installed = probe.compatible && (live || await cachedCommandSucceeds(probe.command, probe.args)); + let ramMB: number | null = null; let vramMB: number | null = null; let device: string | undefined; + if (live && server.process.pid) { + try { ramMB = (await pidusage(server.process.pid)).memory / 1024 / 1024; } catch { /* exited between reads */ } + const gpu = await gpuProcessMemory(server.process.pid); vramMB = gpu.vramMB; device = gpu.device; + if (server.state === "running" && !(await healthCheck(server.baseUrl, server.model))) server.state = "unhealthy"; + } + const snapshot = stoppedSnapshots.get(backend); const issues: string[] = []; + if (!probe.compatible) issues.push(probe.detail); if (probe.compatible && !installed) issues.push(`Runtime executable unavailable: ${probe.command}`); + return { backend, compatible: probe.compatible, installed, running: live && server.state === "running", state: server?.state ?? "stopped", model: server?.model ?? snapshot?.model, + detail: probe.detail, device, pid: server?.process.pid ?? null, port: server?.port ?? null, startedAt: server ? new Date(server.startedAt).toISOString() : null, + uptimeSeconds: server ? Math.max(0, (Date.now() - server.startedAt) / 1000) : 0, ramMB: ramMB === null ? null : +ramMB.toFixed(1), vramMB, + logs: server ? [...server.logs] : snapshot?.logs ?? [], startupError: server?.startupError ?? snapshot?.startupError, + installCommand: installCommand(backend), environmentIssues: issues }; + })); } diff --git a/app/src/main.ts b/app/src/main.ts index bbbc921..74b6bbc 100644 --- a/app/src/main.ts +++ b/app/src/main.ts @@ -14,15 +14,23 @@ import * as secretsStore from "./secrets-store"; import * as dataTransfer from "./data-transfer"; import * as rag from "./rag"; import * as agentTools from "./agent-tools"; +import { detectSandboxCapabilities } from "./command-sandbox"; +import * as terminalManager from "./terminal-manager"; import * as mcpClient from "./mcp-client"; import * as figma from "./figma"; import * as ocr from "./ocr"; import * as huggingface from "./huggingface"; +import * as downloadQueue from "./download-queue"; import * as accounts from "./accounts"; import * as llamacpp from "./llamacpp-manager"; import * as scheduledTasksStore from "./scheduled-tasks-store"; import * as scheduler from "./scheduler"; import * as localServers from "./local-server-manager"; +import * as benchmarkRunner from "./benchmark-runner"; +import * as powerMonitor from "./power-monitor"; +import * as energyUsageStore from "./energy-usage-store"; +import * as pythonRuntimes from "./python-runtime-manager"; +import type { EnergyMonitorSettings } from "./energy-types"; import type { McpServerConfig } from "./mcp-client"; import type { AttachedFile } from "./file-reader"; import * as openaiProvider from "./providers/openai"; @@ -44,6 +52,7 @@ function customProviderSecretKey(customProviderId: string): string { } const activeChatRequests = new Map(); +const activeBenchmarkRequests = new Map(); let isBusy = false; let forceClose = false; @@ -67,6 +76,28 @@ function getLlamaCppModelsDir(): string { return dir; } +function getLocalRuntimeConfig(): localServers.LocalBackendConfig { + const settings = settingsStore.getSettings(); + return { mlxPythonPath: settings.mlxPythonPath, rocmServerPath: settings.rocmServerPath, vllmCommand: settings.vllmCommand }; +} + +function getEnergyMonitorSettings(): EnergyMonitorSettings { + const settings = settingsStore.getSettings(); + return { + enabled: settings.energyMonitoringEnabled ?? false, + electricityPricePerKwh: Math.max(0, settings.electricityPricePerKwh ?? 0.2), + currency: settings.energyCurrency?.trim() || "USD", + timeOfUseTariffs: settings.timeOfUseTariffs ?? [], + manualCpuWatts: settings.manualCpuWatts, + manualGpuWatts: settings.manualGpuWatts, + manualSystemIdleWatts: settings.manualSystemIdleWatts, + includeIdleSystemConsumption: settings.includeIdleSystemConsumption ?? true, + retentionDays: Math.max(1, settings.energyUsageRetentionDays ?? 365), + sampleIntervalSeconds: Math.max(1, Math.min(5, settings.energySampleIntervalSeconds ?? 2)), + gridIntensityGCo2PerKwh: settings.gridIntensityGCo2PerKwh, + }; +} + // Without these, an unexpected error anywhere in the main process (a bad file // parse, a network hiccup, a third-party library throwing) would crash the // entire app instead of just failing the one operation that triggered it. @@ -199,6 +230,23 @@ async function dispatchChat( signal?: AbortSignal, tools?: ToolDefinition[] ): Promise { + const currentSettings = settingsStore.getSettings(); + const customLocal = provider === "custom" + && currentSettings.customProviders?.find((item) => model.startsWith(`${item.id}::`))?.localGpuBackend; + const localProvider = ["ollama", "llamacpp", "mlx", "rocm", "vllm"].includes(provider) || !!customLocal; + const energySettings = getEnergyMonitorSettings(); + energySettings.enabled = energySettings.enabled && localProvider; + const backend = provider === "llamacpp" + ? currentSettings.llamaCppGpuBackend ?? "auto" + : provider === "rocm" ? "rocm" : provider; + const initialPromptTokens = Math.max(1, Math.ceil(messages.reduce((sum, message) => sum + message.content.length, 0) / 4)); + const activity = powerMonitor.beginRequest(provider, model, backend, energySettings, initialPromptTokens); + const downstreamToken = onToken; + onToken = (chunk) => { + activity.onChunk(chunk); + downstreamToken(chunk); + }; + try { if (provider === "ollama") { await ollama.chat(model, messages, options, onToken, signal, tools); } else if (provider === "llamacpp") { @@ -279,6 +327,9 @@ async function dispatchChat( provider === "openai" ? openaiProvider.chat : provider === "anthropic" ? anthropicProvider.chat : geminiProvider.chat; await providerFn(apiKey, model, messages, options, onToken, signal, tools); } + } finally { + await activity.finish(); + } } // Runs a single-turn prompt to completion and returns the full text — @@ -292,6 +343,79 @@ async function completePrompt(provider: ProviderId, model: string, prompt: strin return text; } +function benchmarkResultPath(): string { + const dir = path.join(app.getPath("userData"), "benchmarks"); + fs.mkdirSync(dir, { recursive: true }); + return path.join(dir, "latest.json"); +} + +function benchmarkHistoryPath(): string { return path.join(path.dirname(benchmarkResultPath()), "history.json"); } + +function saveBenchmarkResult(result: benchmarkRunner.BenchmarkResult): void { + const destination = benchmarkResultPath(); + const temporary = `${destination}.tmp`; + fs.writeFileSync(temporary, JSON.stringify(result, null, 2)); + fs.renameSync(temporary, destination); + let history: benchmarkRunner.BenchmarkResult[] = []; + try { history = JSON.parse(fs.readFileSync(benchmarkHistoryPath(), "utf-8")); } catch { /* first benchmark */ } + history.push(result); + fs.writeFileSync(benchmarkHistoryPath(), JSON.stringify(history.slice(-100), null, 2)); +} + +function getBenchmarkObservations(): systemSpecs.BenchmarkObservation[] { + try { + const history = JSON.parse(fs.readFileSync(benchmarkHistoryPath(), "utf-8")) as benchmarkRunner.BenchmarkResult[]; + return history.flatMap((result) => result.primary ? [{ model: result.model, tokensPerSecond: result.primary.tokensPerSecond, promptTokensPerSecond: result.primary.promptTokensPerSecond, timeToFirstTokenMs: result.primary.timeToFirstTokenMs }] : []); + } catch { + const latest = getLastBenchmarkResult(); + return latest?.primary ? [{ model: latest.model, tokensPerSecond: latest.primary.tokensPerSecond, promptTokensPerSecond: latest.primary.promptTokensPerSecond, timeToFirstTokenMs: latest.primary.timeToFirstTokenMs }] : []; + } +} + +function getLastBenchmarkResult(): benchmarkRunner.BenchmarkResult | null { + try { + return JSON.parse(fs.readFileSync(benchmarkResultPath(), "utf-8")) as benchmarkRunner.BenchmarkResult; + } catch { + return null; + } +} + +async function exportDiagnosticReport(result: benchmarkRunner.BenchmarkResult): Promise<{ success: boolean }> { + const specs = await systemSpecs.getSpecs(); + const settings = settingsStore.getSettings(); + const runtimes = await localServers.getRuntimeStatuses({ + mlxPythonPath: settings.mlxPythonPath, + rocmServerPath: settings.rocmServerPath, + vllmCommand: settings.vllmCommand, + }); + const report = { + schemaVersion: 1, + exportedAt: new Date().toISOString(), + application: { + version: app.getVersion(), + electron: process.versions.electron, + chrome: process.versions.chrome, + node: process.versions.node, + platform: process.platform, + arch: process.arch, + }, + system: specs, + runtimeHealth: runtimes, + benchmark: result, + energyUsage: powerMonitor.getDashboard(getEnergyMonitorSettings()), + recentLogs: getLogTail(), + }; + const date = new Date().toISOString().replace(/[:.]/g, "-"); + const options = { + defaultPath: `modelforge-diagnostic-${date}.json`, + filters: [{ name: "JSON", extensions: ["json"] }], + }; + const dialogResult = mainWindow ? await dialog.showSaveDialog(mainWindow, options) : await dialog.showSaveDialog(options); + if (dialogResult.canceled || !dialogResult.filePath) return { success: false }; + fs.writeFileSync(dialogResult.filePath, JSON.stringify(report, null, 2)); + return { success: true }; +} + function registerIpcHandlers(): void { ipcMain.handle("ollama:status", () => ollama.isRunning()); ipcMain.handle("ollama:start", () => ollama.start()); @@ -335,13 +459,17 @@ function registerIpcHandlers(): void { }); ipcMain.handle("llamacpp:getAvailableGpuBackends", () => llamacpp.getAvailableGpuBackends()); ipcMain.handle("localBackends:getStatuses", () => { - const settings = settingsStore.getSettings(); - return localServers.getRuntimeStatuses({ - mlxPythonPath: settings.mlxPythonPath, - rocmServerPath: settings.rocmServerPath, - vllmCommand: settings.vllmCommand, - }); + return localServers.getRuntimeStatuses(getLocalRuntimeConfig()); }); + ipcMain.handle("localBackends:start", (_event, { backend, model }: { backend: localServers.LocalBackendId; model: string }) => + localServers.startServer(backend, requireString(model, "runtime model"), getLocalRuntimeConfig()) + ); + ipcMain.handle("localBackends:stop", (_event, backend: localServers.LocalBackendId) => localServers.stopServer(backend)); + ipcMain.handle("localBackends:restart", (_event, { backend, model }: { backend: localServers.LocalBackendId; model: string }) => + localServers.restartServer(backend, requireString(model, "runtime model"), getLocalRuntimeConfig()) + ); + ipcMain.handle("localBackends:unload", (_event, backend: localServers.LocalBackendId) => localServers.stopServer(backend)); + ipcMain.handle("pythonRuntimes:getStatuses", () => pythonRuntimes.getPythonEnvironmentStatuses()); ipcMain.handle("llamacpp:setGpuBackend", async (_event: IpcMainInvokeEvent, backend: llamacpp.GpuBackend) => { await llamacpp.setGpuBackend(backend); settingsStore.saveSettings({ llamaCppGpuBackend: backend }); @@ -417,7 +545,8 @@ function registerIpcHandlers(): void { ipcMain.handle("system:getSpecs", () => systemSpecs.getSpecs()); ipcMain.handle("system:getRecommendations", async () => { const specs = await systemSpecs.getSpecs(); - return systemSpecs.recommendModels(specs); + const settings = settingsStore.getSettings(); + return systemSpecs.recommendModelsWithML(specs, { contextLength: settings.contextLength, quantization: "Q4_K_M", runtime: settings.preferredRuntime ?? "automatic" }, getBenchmarkObservations()); }); ipcMain.handle("system:getActivity", async () => { const ollamaRunning = await ollama.isRunning(); @@ -435,11 +564,81 @@ function registerIpcHandlers(): void { }; }); + ipcMain.handle( + "benchmark:run", + async ( + _event: IpcMainInvokeEvent, + { requestId, request }: { requestId: string; request: benchmarkRunner.BenchmarkRequest } + ) => { + requireString(requestId, "benchmark request id"); + requireString(request?.provider, "benchmark provider"); + requireString(request?.model, "benchmark model"); + const controller = new AbortController(); + activeBenchmarkRequests.set(requestId, controller); + try { + const result = await benchmarkRunner.runBenchmark( + (provider, model, messages, options, onToken, signal) => + dispatchChat(provider, model, messages, options, onToken, signal), + request, + controller.signal + ); + saveBenchmarkResult(result); + return { result }; + } catch (error) { + if ((error as Error).name === "AbortError") return { aborted: true }; + logger.error(`Benchmark failed (provider=${request.provider}, model=${request.model}): ${(error as Error).message}`); + return { error: (error as Error).message }; + } finally { + activeBenchmarkRequests.delete(requestId); + } + } + ); + ipcMain.handle("benchmark:cancel", (_event: IpcMainInvokeEvent, requestId: string) => { + activeBenchmarkRequests.get(requireString(requestId, "benchmark request id"))?.abort(); + }); + ipcMain.handle("benchmark:getLast", () => getLastBenchmarkResult()); + ipcMain.handle("benchmark:exportReport", (_event: IpcMainInvokeEvent, result: benchmarkRunner.BenchmarkResult) => + exportDiagnosticReport(result) + ); + ipcMain.handle("energy:getDashboard", () => powerMonitor.getDashboard(getEnergyMonitorSettings())); + ipcMain.handle("energy:clearHistory", () => { + energyUsageStore.clearRecords(); + return { success: true }; + }); + + ipcMain.handle("downloads:list", () => downloadQueue.getJobs()); + ipcMain.handle("downloads:create", (_event, input: { modelId: string; filename: string; expectedBytes: number; backend?: Parameters[0]["backend"] }) => + downloadQueue.createHuggingFaceJob({ + ...input, + backend: input.backend ?? settingsStore.getSettings().preferredRuntime ?? "automatic", + destinationDir: getLlamaCppModelsDir(), + }) + ); + ipcMain.handle("downloads:pause", (_event, id: string) => downloadQueue.pauseJob(requireString(id, "download id"))); + ipcMain.handle("downloads:resume", (_event, id: string) => downloadQueue.resumeJob(requireString(id, "download id"))); + ipcMain.handle("downloads:retry", (_event, id: string) => downloadQueue.retryJob(requireString(id, "download id"))); + ipcMain.handle("downloads:cancel", (_event, id: string) => downloadQueue.cancelJob(requireString(id, "download id"))); + ipcMain.handle("downloads:delete", (_event, id: string) => downloadQueue.removeJob(requireString(id, "download id"))); + ipcMain.handle("downloads:forecast", (_event, id: string) => downloadQueue.diskForecast(requireString(id, "download id"))); + ipcMain.handle("downloads:recoveryStatus", () => downloadQueue.getRecoveryStatus()); + ipcMain.handle("downloads:getControls", () => { + const settings = settingsStore.getSettings(); + return { concurrency: settings.downloadGlobalConcurrency ?? 2, bandwidthMbps: settings.downloadBandwidthMbps ?? 0 }; + }); + ipcMain.handle("downloads:setControls", (_event, controls: downloadQueue.DownloadControls) => { + const normalized = downloadQueue.configure(controls); + settingsStore.saveSettings({ downloadGlobalConcurrency: normalized.concurrency, downloadBandwidthMbps: normalized.bandwidthMbps }); + return normalized; + }); + ipcMain.handle("settings:get", () => settingsStore.getSettings()); ipcMain.handle("settings:save", (_event: IpcMainInvokeEvent, partial) => { const saved = settingsStore.saveSettings(partial); if (partial.ollamaHost !== undefined) ollama.setHost(saved.ollamaHost); if (partial.llamaCppMaxCachedModels !== undefined) llamacpp.setModelCacheLimit(saved.llamaCppMaxCachedModels ?? 2); + if (partial.downloadGlobalConcurrency !== undefined || partial.downloadBandwidthMbps !== undefined) { + downloadQueue.configure({ concurrency: saved.downloadGlobalConcurrency ?? 2, bandwidthMbps: saved.downloadBandwidthMbps ?? 0 }); + } if (partial.keybindings !== undefined) { setupMenu(() => mainWindow, () => checkForUpdatesManually(() => mainWindow), saved.keybindings); } @@ -539,6 +738,7 @@ function registerIpcHandlers(): void { ipcMain.handle("secrets:set", (_event: IpcMainInvokeEvent, { key, value }: { key: string; value: string }) => secretsStore.setSecret(requireString(key, "secret key"), value ?? "") ); + ipcMain.handle("secrets:isEncryptionAvailable", () => secretsStore.isEncryptionAvailable()); ipcMain.handle("accounts:status", (_event: IpcMainInvokeEvent, provider: accounts.AccountProvider) => accounts.getLinkedAccount(provider) @@ -609,12 +809,18 @@ function registerIpcHandlers(): void { ); ipcMain.handle("data:importPromptPresets", () => dataTransfer.importPromptPresets(mainWindow)); - ipcMain.handle("rag:indexFiles", (_event: IpcMainInvokeEvent, files: AttachedFile[]) => rag.indexFiles(files)); + ipcMain.handle( + "rag:indexFolder", + (_event: IpcMainInvokeEvent, input: { folderPath: string; folderName: string; files: AttachedFile[] }) => + rag.indexFolder({ ...input, embeddingModel: settingsStore.getSettings().ragEmbeddingModel }) + ); ipcMain.handle( "rag:query", - (_event: IpcMainInvokeEvent, { indexId, query, topK }: { indexId: string; query: string; topK?: number }) => - rag.query(indexId, query, topK) + (_event: IpcMainInvokeEvent, { collectionId, query, topK }: { collectionId: string; query: string; topK?: number }) => + rag.query(collectionId, query, topK) ); + ipcMain.handle("rag:listCollections", () => rag.listCollections()); + ipcMain.handle("rag:deleteCollection", (_event: IpcMainInvokeEvent, id: string) => rag.deleteCollection(requireString(id, "collection id"))); ipcMain.handle("hf:search", async (_event: IpcMainInvokeEvent, query: string) => { try { @@ -754,6 +960,58 @@ function registerIpcHandlers(): void { return agentTools.detectProjectScripts(workspaceRoot); }); + // Called when the renderer is about to stop using a workspace (switching + // to a different folder, or loading a session that points elsewhere) — + // without this, background tasks started against the old workspace kept + // running indefinitely, since killAllBackgroundCommands() only ever ran + // on app quit. + ipcMain.handle("agent:closeWorkspace", (_event: IpcMainInvokeEvent, workspaceRoot: string) => { + requireString(workspaceRoot, "workspace root"); + const killedBackgroundTasks = agentTools.killBackgroundCommandsForWorkspace(workspaceRoot); + const killedTerminals = terminalManager.closeAllForWorkspace(workspaceRoot); + return { killedBackgroundTasks, killedTerminals }; + }); + + ipcMain.handle("agent:getSandboxCapabilities", () => detectSandboxCapabilities()); + + ipcMain.handle( + "terminal:create", + ( + event: IpcMainInvokeEvent, + { workspaceRoot, opts }: { workspaceRoot: string; opts?: { cwd?: string; name?: string } } + ) => { + requireString(workspaceRoot, "workspace root"); + // `id` is referenced inside these callbacks before it's assigned + // below, but that's safe: node-pty never calls onData/onExit + // synchronously during createTerminal() itself, only later once + // its own async event loop runs — by which point `id` is set. + const { id, name } = terminalManager.createTerminal( + workspaceRoot, + opts ?? {}, + (chunk) => event.sender.send(`terminal:data:${id}`, chunk), + (exitCode) => event.sender.send(`terminal:exit:${id}`, exitCode) + ); + return { id, name }; + } + ); + + ipcMain.handle("terminal:write", (_event: IpcMainInvokeEvent, { id, data }: { id: string; data: string }) => { + requireString(id, "terminal id"); + terminalManager.writeToTerminal(id, typeof data === "string" ? data : ""); + }); + + ipcMain.handle("terminal:resize", (_event: IpcMainInvokeEvent, { id, cols, rows }: { id: string; cols: number; rows: number }) => { + requireString(id, "terminal id"); + terminalManager.resizeTerminal(id, Number(cols) || 80, Number(rows) || 24); + }); + + ipcMain.handle("terminal:close", (_event: IpcMainInvokeEvent, id: string) => { + requireString(id, "terminal id"); + terminalManager.closeTerminal(id); + }); + + ipcMain.handle("terminal:list", (_event: IpcMainInvokeEvent, workspaceRoot?: string) => terminalManager.listTerminals(workspaceRoot)); + ipcMain.handle("mcp:connect", async (_event: IpcMainInvokeEvent, config: McpServerConfig) => { try { const { tools } = await mcpClient.connectServer(config); @@ -798,6 +1056,12 @@ app.whenReady().then(async () => { llamacpp.setModelCacheLimit(settingsStore.getSettings().llamaCppMaxCachedModels ?? 2); await llamacpp.setGpuBackend(settingsStore.getSettings().llamaCppGpuBackend ?? "auto"); setupAutoUpdater(() => mainWindow); + downloadQueue.init(() => mainWindow); + downloadQueue.configure({ + concurrency: settingsStore.getSettings().downloadGlobalConcurrency ?? 2, + bandwidthMbps: settingsStore.getSettings().downloadBandwidthMbps ?? 0, + }); + void downloadQueue.resumeInterruptedJobs(); void connectEnabledMcpServers(); scheduler.init((provider, model, prompt) => completePrompt(provider as ProviderId, model, prompt)); @@ -810,7 +1074,9 @@ app.on("window-all-closed", () => { ollama.stop(); localServers.stopAll(); agentTools.killAllBackgroundCommands(); + terminalManager.closeAll(); mcpClient.disconnectAll(); + powerMonitor.stopAll(); if (process.platform !== "darwin") app.quit(); }); @@ -818,5 +1084,7 @@ app.on("before-quit", () => { ollama.stop(); localServers.stopAll(); agentTools.killAllBackgroundCommands(); + terminalManager.closeAll(); void llamacpp.dispose(); + powerMonitor.stopAll(); }); diff --git a/app/src/media.ts b/app/src/media.ts index f1ad8df..f3e6570 100644 --- a/app/src/media.ts +++ b/app/src/media.ts @@ -53,6 +53,26 @@ export async function extractPdfText(filePath: string): Promise { } } +export interface PdfPage { + num: number; + text: string; +} + +// Same extraction as extractPdfText, but keeps text broken down per page — +// getText() already returns this (TextResult.pages), no extra per-page calls +// needed. Used by RAG folder-indexing to attach page numbers to chunks. +export async function extractPdfPages(filePath: string): Promise<{ text: string; pages: PdfPage[] }> { + const { PDFParse } = require("pdf-parse"); + const buffer = fs.readFileSync(filePath); + const parser = new PDFParse({ data: buffer }); + try { + const result = await parser.getText(); + return { text: result.text as string, pages: (result.pages ?? []).map((p: { num: number; text: string }) => ({ num: p.num, text: p.text })) }; + } finally { + await parser.destroy(); + } +} + function ffmpegBinaryPath(): string { // electron-builder's asarUnpack keeps this binary outside the asar archive // (asar-packed files can't be exec'd directly); this swap finds it there diff --git a/app/src/native-downloader.ts b/app/src/native-downloader.ts new file mode 100644 index 0000000..683ffee --- /dev/null +++ b/app/src/native-downloader.ts @@ -0,0 +1,99 @@ +import * as path from "node:path"; + +export interface NativeDownloadProgress { + receivedBytes: number; + totalBytes?: number; +} + +export interface JobEvent { + jobId: string; + kind: "shard_progress" | "shard_state" | "job_state" | "job_error"; + shardFilename?: string; + shardState?: string; + jobState?: string; + receivedBytes?: number; + jobReceivedBytes?: number; + totalBytes?: number; + bytesPerSec?: number; + etaSeconds?: number; + errorMessage?: string; + errorKind?: string; + retryable?: boolean; +} + +export interface JsShard { + filename: string; + path: string; + expectedBytes: number; + receivedBytes: number; + sha256?: string; +} + +export interface JsDownloadJob { + id: string; + modelId: string; + token?: string; + destinationDir: string; + shards: JsShard[]; +} + +export interface NativeDownloadManager { + setGlobalConcurrency(limit: number): void; + setBandwidthLimit(bytesPerSec: number | undefined | null): void; + pauseJob(jobId: string): void; + cancelJob(jobId: string): void; + startJob(job: JsDownloadJob, onEvent: (err: Error | null, event: JobEvent) => void): Promise; +} + +interface NativeAddon { + downloadGgufFile( + modelId: string, + filename: string, + destPath: string, + token: string | undefined | null, + onProgress: (err: Error | null, progress: NativeDownloadProgress) => void + ): Promise; + DownloadManager: new () => NativeDownloadManager; +} + +// The Rust addon is built by `lib`'s `napi build` step into app/native/ as a +// plain filesystem artifact (not an npm dependency under node_modules — +// see the plan for why: a `file:` dependency would need `npm ci` to +// reconcile a different lockfile integrity hash per platform's .node +// binary). `dist/native-downloader.js` sits one level under app/, so +// app/native is a sibling reached by going up just one directory. Electron +// transparently redirects native-module requires from inside app.asar to +// the asarUnpack'd app.asar.unpacked counterpart at packaging time. +let nativeAddon: NativeAddon | undefined; + +function getNativeAddon(): NativeAddon { + if (!nativeAddon) { + nativeAddon = require(path.join(__dirname, "..", "native")) as NativeAddon; + } + return nativeAddon; +} + +export function downloadGgufFileNative( + modelId: string, + filename: string, + destPath: string, + token: string | undefined | null, + onProgress: (err: Error | null, progress: NativeDownloadProgress) => void +): Promise { + return getNativeAddon().downloadGgufFile(modelId, filename, destPath, token, onProgress); +} + +// One instance for the app's lifetime — its global concurrency semaphore +// and bandwidth limiter are only meaningfully "global" if nothing else +// constructs a second one. Purely additive to `downloadGgufFileNative` +// above, which stays exactly as-is for the existing single-file `hf: +// downloadFile` flow. +let downloadManager: NativeDownloadManager | undefined; + +export function getDownloadManager(): NativeDownloadManager { + if (!downloadManager) { + const { DownloadManager } = getNativeAddon(); + downloadManager = new DownloadManager(); + } + return downloadManager; +} diff --git a/app/src/power-monitor.test.ts b/app/src/power-monitor.test.ts new file mode 100644 index 0000000..15eb231 --- /dev/null +++ b/app/src/power-monitor.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { estimatePower, intervalEnergyKwh, perActivityEnergyKwh } from "./power-monitor"; + +describe("power estimation", () => { + it("interpolates between idle and configured maximum wattage", () => { + const low = estimatePower(0, 0, { manualCpuWatts: 100, manualGpuWatts: 200, manualSystemIdleWatts: 20, includeIdleSystemConsumption: true }); + const high = estimatePower(1, 1, { manualCpuWatts: 100, manualGpuWatts: 200, manualSystemIdleWatts: 20, includeIdleSystemConsumption: true }); + expect(low).toBeGreaterThanOrEqual(20); + expect(high).toBeCloseTo(320); + expect(high).toBeGreaterThan(low); + }); +}); + +describe("shared energy allocation", () => { + it("does not double-count one machine interval across concurrent requests", () => { + const total = intervalEnergyKwh(200, 60_000); + const perRequest = perActivityEnergyKwh(200, 60_000, 2); + expect(perRequest * 2).toBeCloseTo(total, 12); + }); +}); diff --git a/app/src/power-monitor.ts b/app/src/power-monitor.ts new file mode 100644 index 0000000..70f520a --- /dev/null +++ b/app/src/power-monitor.ts @@ -0,0 +1,334 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { randomUUID } from "node:crypto"; +import type { ChatChunk, ProviderId } from "./providers/types"; +import * as usageStore from "./energy-usage-store"; +import type { + EnergyDashboard, + EnergyMeasurement, + EnergyMonitorSettings, + EnergyRuntime, + EnergyTotals, + EnergyUsageRecord, + RuntimeActivity, + TariffUsageSnapshot, +} from "./energy-types"; + +const execFileAsync = promisify(execFile); +const active = new Map(); +let previousCpu = cpuTimes(); +let raplPrevious: { energyUj: number; at: number } | null = null; +let samplerTimer: NodeJS.Timeout | null = null; +let samplePromise: Promise | null = null; +let lastSharedSampleAt = Date.now(); + +interface PowerReading { + watts: number; + cpuUtilization: number; + gpuUtilization: number | null; + measurement: EnergyMeasurement; +} + +interface InternalActivity extends RuntimeActivity { + firstTokenAt?: number; + tariffSnapshots: Map; + gridIntensity: number; + settings: EnergyMonitorSettings; +} + +export interface ActivityHandle { + onChunk(chunk: ChatChunk): void; + finish(): Promise; +} + +function cpuTimes(): { idle: number; total: number } { + let idle = 0; + let total = 0; + for (const cpu of os.cpus()) { + idle += cpu.times.idle; + total += Object.values(cpu.times).reduce((sum, value) => sum + value, 0); + } + return { idle, total }; +} + +export function sampleCpuUtilization(): number { + const current = cpuTimes(); + const totalDelta = current.total - previousCpu.total; + const idleDelta = current.idle - previousCpu.idle; + previousCpu = current; + return totalDelta <= 0 ? 0 : Math.max(0, Math.min(1, 1 - idleDelta / totalDelta)); +} + +function runtimeForProvider(provider: ProviderId): EnergyRuntime { + if (provider === "ollama") return "ollama"; + if (provider === "vllm") return "vllm"; + if (provider === "mlx") return "mlx"; + if (provider === "llamacpp" || provider === "rocm") return "llamacpp"; + return "transformers"; +} + +function selectedTariff(settings: EnergyMonitorSettings, date: Date): { name: string; pricePerKwh: number } { + const hour = date.getHours() + date.getMinutes() / 60; + const match = settings.timeOfUseTariffs.find((tariff) => + tariff.startHour <= tariff.endHour + ? hour >= tariff.startHour && hour < tariff.endHour + : hour >= tariff.startHour || hour < tariff.endHour + ); + return match ?? { name: "Standard", pricePerKwh: settings.electricityPricePerKwh }; +} + +function readRaplEnergy(): number | null { + if (process.platform !== "linux") return null; + const roots = ["/sys/class/powercap/intel-rapl:0/energy_uj", "/sys/class/powercap/intel-rapl/intel-rapl:0/energy_uj"]; + for (const candidate of roots) { + try { + const value = Number(fs.readFileSync(candidate, "utf-8").trim()); + if (Number.isFinite(value)) return value; + } catch { + // Try the next common RAPL location. + } + } + return null; +} + +function raplWatts(now: number): number | null { + const energyUj = readRaplEnergy(); + if (energyUj === null) return null; + const previous = raplPrevious; + raplPrevious = { energyUj, at: now }; + if (!previous || energyUj < previous.energyUj || now <= previous.at) return null; + return ((energyUj - previous.energyUj) / 1e6) / ((now - previous.at) / 1000); +} + +async function nvidiaReading(): Promise<{ watts: number; utilization: number } | null> { + try { + const { stdout } = await execFileAsync("nvidia-smi", ["--query-gpu=power.draw,utilization.gpu", "--format=csv,noheader,nounits"], { timeout: 2500, windowsHide: true }); + let watts = 0; + let utilization = 0; + let count = 0; + for (const line of stdout.split(/\r?\n/)) { + const [power, usage] = line.split(",").map((value) => Number(value.trim())); + if (!Number.isFinite(power)) continue; + watts += power; + utilization += Number.isFinite(usage) ? usage / 100 : 0; + count++; + } + return count ? { watts, utilization: utilization / count } : null; + } catch { + return null; + } +} + +async function rocmReading(): Promise<{ watts: number; utilization: number } | null> { + try { + const { stdout } = await execFileAsync("rocm-smi", ["--showpower", "--showuse", "--json"], { timeout: 2500, windowsHide: true }); + const parsed = JSON.parse(stdout) as Record>; + let watts = 0; + let utilization = 0; + let count = 0; + for (const card of Object.values(parsed)) { + const powerEntry = Object.entries(card).find(([key]) => /average.*power|power.*average/i.test(key)); + const useEntry = Object.entries(card).find(([key]) => /gpu.*use/i.test(key)); + if (!powerEntry) continue; + const power = Number(String(powerEntry[1]).replace(/[^0-9.]/g, "")); + if (!Number.isFinite(power)) continue; + watts += power > 1000 ? power / 1e6 : power; + utilization += Number(String(useEntry?.[1] ?? 0).replace(/[^0-9.]/g, "")) / 100; + count++; + } + return count ? { watts, utilization: utilization / count } : null; + } catch { + return null; + } +} + +async function appleReading(): Promise<{ cpuWatts: number; gpuWatts: number } | null> { + if (process.platform !== "darwin") return null; + try { + const { stdout } = await execFileAsync("powermetrics", ["--samplers", "cpu_power,gpu_power", "-i", "500", "-n", "1"], { timeout: 2500, windowsHide: true }); + const cpu = Number(stdout.match(/CPU Power:\s*([0-9.]+)\s*mW/i)?.[1] ?? NaN) / 1000; + const gpu = Number(stdout.match(/GPU Power:\s*([0-9.]+)\s*mW/i)?.[1] ?? NaN) / 1000; + return Number.isFinite(cpu) || Number.isFinite(gpu) ? { cpuWatts: Number.isFinite(cpu) ? cpu : 0, gpuWatts: Number.isFinite(gpu) ? gpu : 0 } : null; + } catch { + return null; + } +} + +export function estimatePower( + cpuUtilization: number, + gpuUtilization: number, + settings: Pick +): number { + const idle = settings.includeIdleSystemConsumption ? (settings.manualSystemIdleWatts ?? 25) : 0; + const cpuMax = settings.manualCpuWatts ?? 65; + const gpuMax = settings.manualGpuWatts ?? 150; + const cpuIdle = Math.min(10, cpuMax * 0.2); + const gpuIdle = Math.min(12, gpuMax * 0.12); + return idle + + cpuIdle + cpuUtilization * Math.max(0, cpuMax - cpuIdle) + + gpuIdle + gpuUtilization * Math.max(0, gpuMax - gpuIdle); +} + +async function readPower(settings: EnergyMonitorSettings): Promise { + const now = Date.now(); + const cpuUtilization = sampleCpuUtilization(); + const apple = process.platform === "darwin" ? await appleReading() : null; + const gpu = apple ? null : (await nvidiaReading()) ?? (await rocmReading()); + const cpuWatts = raplWatts(now); + if (apple) { + return { watts: apple.cpuWatts + apple.gpuWatts + (settings.includeIdleSystemConsumption ? settings.manualSystemIdleWatts ?? 0 : 0), cpuUtilization, gpuUtilization: null, measurement: "measured" }; + } + if (cpuWatts !== null && gpu) { + return { watts: cpuWatts + gpu.watts + (settings.includeIdleSystemConsumption ? settings.manualSystemIdleWatts ?? 0 : 0), cpuUtilization, gpuUtilization: gpu.utilization, measurement: "measured" }; + } + if (gpu) { + // Preserve the hardware GPU reading and estimate only the CPU/system + // portion. The combined result is still labelled Estimated because + // it contains a modelled component. + const estimatedCpuAndSystem = estimatePower(cpuUtilization, 0, { ...settings, manualGpuWatts: 0 }); + return { watts: estimatedCpuAndSystem + gpu.watts, cpuUtilization, gpuUtilization: gpu.utilization, measurement: "estimated" }; + } + if (cpuWatts !== null) { + return { watts: cpuWatts + (settings.includeIdleSystemConsumption ? settings.manualSystemIdleWatts ?? 0 : 0), cpuUtilization, gpuUtilization: null, measurement: "measured" }; + } + return { watts: estimatePower(cpuUtilization, 0, settings), cpuUtilization, gpuUtilization: null, measurement: "estimated" }; +} + +export function intervalEnergyKwh(watts: number, elapsedMs: number): number { + return watts * (Math.max(0, elapsedMs) / 3_600_000) / 1000; +} + +export function perActivityEnergyKwh(watts: number, elapsedMs: number, activityCount: number): number { + return activityCount > 0 ? intervalEnergyKwh(watts, elapsedMs) / activityCount : 0; +} + +async function sampleAll(): Promise { + if (samplePromise) return samplePromise; + if (active.size === 0) return; + samplePromise = (async () => { + const settings = active.values().next().value?.settings as EnergyMonitorSettings | undefined; + if (!settings) return; + const reading = await readPower(settings); + const now = Date.now(); + const elapsedMs = Math.max(0, now - lastSharedSampleAt); + lastSharedSampleAt = now; + const activities = [...active.values()]; + const shareWatts = reading.watts / activities.length; + const energyKwh = perActivityEnergyKwh(reading.watts, elapsedMs, activities.length); + for (const activity of activities) { + const tariff = selectedTariff(activity.settings, new Date(now)); + const cost = energyKwh * tariff.pricePerKwh; + activity.currentPowerWatts = shareWatts; + activity.cpuUtilization = reading.cpuUtilization; + activity.gpuUtilization = reading.gpuUtilization; + activity.energyKwh += energyKwh; + activity.cost += cost; + if (reading.measurement === "estimated") activity.measurement = "estimated"; + const key = `${tariff.name}\u0000${tariff.pricePerKwh}\u0000${activity.settings.currency}`; + const bucket = activity.tariffSnapshots.get(key) ?? { name: tariff.name, pricePerKwh: tariff.pricePerKwh, currency: activity.settings.currency, energyKwh: 0, cost: 0 }; + bucket.energyKwh += energyKwh; + bucket.cost += cost; + activity.tariffSnapshots.set(key, bucket); + } + })().finally(() => { samplePromise = null; }); + return samplePromise; +} + +function ensureSampler(settings: EnergyMonitorSettings): void { + if (samplerTimer) return; + lastSharedSampleAt = Date.now(); + void sampleAll(); + samplerTimer = setInterval(() => void sampleAll(), Math.max(1, Math.min(5, settings.sampleIntervalSeconds)) * 1000); + samplerTimer.unref(); +} + +function stopSamplerIfIdle(): void { + if (active.size !== 0 || !samplerTimer) return; + clearInterval(samplerTimer); + samplerTimer = null; +} + +export function beginRequest(provider: ProviderId, modelId: string, backend: string, settings: EnergyMonitorSettings, initialPromptTokens = 0): ActivityHandle { + if (!settings.enabled) return { onChunk: () => undefined, finish: async () => undefined }; + const now = Date.now(); + const activity: InternalActivity = { + id: randomUUID(), runtime: runtimeForProvider(provider), modelId, backend, + device: backend, processId: provider === "llamacpp" ? process.pid : null, + startedAt: new Date(now).toISOString(), promptTokens: initialPromptTokens, completionTokens: 0, + activeSeconds: 0, loadingSeconds: 0, cpuUtilization: 0, gpuUtilization: null, + currentPowerWatts: 0, energyKwh: 0, cost: 0, measurement: "measured", + tariffSnapshots: new Map(), gridIntensity: settings.gridIntensityGCo2PerKwh ?? 0, settings, + }; + active.set(activity.id, activity); + ensureSampler(settings); + return { + onChunk(chunk) { + if ((chunk.message?.content ?? "") && !activity.firstTokenAt) activity.firstTokenAt = Date.now(); + activity.promptTokens = chunk.usage?.promptTokens ?? activity.promptTokens; + activity.completionTokens = chunk.usage?.completionTokens ?? activity.completionTokens; + if (!chunk.usage?.completionTokens && chunk.message?.content) activity.completionTokens += Math.ceil(chunk.message.content.length / 4); + }, + async finish() { + await sampleAll(); + const finished = Date.now(); + activity.finishedAt = new Date(finished).toISOString(); + activity.loadingSeconds = ((activity.firstTokenAt ?? finished) - now) / 1000; + activity.activeSeconds = activity.firstTokenAt ? (finished - activity.firstTokenAt) / 1000 : 0; + const record: EnergyUsageRecord = { + date: activity.startedAt.slice(0, 10), runtime: activity.runtime, modelId: activity.modelId, + requestCount: 1, promptTokens: activity.promptTokens, completionTokens: activity.completionTokens, + activeSeconds: activity.activeSeconds, loadingSeconds: activity.loadingSeconds, + energyKwh: activity.energyKwh, cost: activity.cost, measurement: activity.measurement, + tariffSnapshots: [...activity.tariffSnapshots.values()], carbonGrams: activity.energyKwh * activity.gridIntensity, + }; + usageStore.addRecord(record, settings); + active.delete(activity.id); + stopSamplerIfIdle(); + }, + }; +} + +function zeroTotals(): EnergyTotals { + return { energyKwh: 0, cost: 0, carbonGrams: 0, requestCount: 0, promptTokens: 0, completionTokens: 0 }; +} + +function add(totals: EnergyTotals, record: EnergyUsageRecord): void { + totals.energyKwh += record.energyKwh; totals.cost += record.cost; totals.carbonGrams += record.carbonGrams; + totals.requestCount += record.requestCount; totals.promptTokens += record.promptTokens; totals.completionTokens += record.completionTokens; +} + +export function getDashboard(settings: EnergyMonitorSettings): EnergyDashboard { + const records = usageStore.listRecords(settings.retentionDays); + const now = new Date(); + const todayKey = now.toISOString().slice(0, 10); + const weekStart = new Date(now); weekStart.setDate(now.getDate() - 6); + const monthStart = new Date(now.getFullYear(), now.getMonth(), 1); + const today = zeroTotals(), week = zeroTotals(), month = zeroTotals(), lifetime = zeroTotals(); + const models = new Map(), runtimes = new Map(); + let measuredEnergy = 0; + for (const record of records) { + add(lifetime, record); + if (record.date === todayKey) add(today, record); + if (record.date >= weekStart.toISOString().slice(0, 10)) add(week, record); + if (record.date >= monthStart.toISOString().slice(0, 10)) add(month, record); + const model = models.get(record.modelId) ?? zeroTotals(); add(model, record); models.set(record.modelId, model); + const runtime = runtimes.get(record.runtime) ?? zeroTotals(); add(runtime, record); runtimes.set(record.runtime, runtime); + if (record.measurement === "measured") measuredEnergy += record.energyKwh; + } + const sorted = (map: Map) => [...map].map(([key, totals]) => ({ key, totals })).sort((a, b) => b.totals.cost - a.totals.cost); + return { + current: [...active.values()].map(({ tariffSnapshots: _tariffs, settings: _settings, firstTokenAt: _first, gridIntensity: _grid, ...item }) => item), + today, week, month, lifetime, byModel: sorted(models), byRuntime: sorted(runtimes), + measuredPercent: lifetime.energyKwh > 0 ? measuredEnergy / lifetime.energyKwh * 100 : 0, + costPerMillionGeneratedTokens: lifetime.completionTokens > 0 ? lifetime.cost / lifetime.completionTokens * 1_000_000 : 0, + currency: settings.currency, records, + }; +} + +export function stopAll(): void { + active.clear(); + if (samplerTimer) clearInterval(samplerTimer); + samplerTimer = null; +} diff --git a/app/src/preload.ts b/app/src/preload.ts index 272d7ee..b0aa69c 100644 --- a/app/src/preload.ts +++ b/app/src/preload.ts @@ -4,6 +4,8 @@ import type { AttachedFile, MediaAttachment } from "./file-reader"; import type { ChatMessage, ChatChunk, ChatOptions, ProviderId } from "./providers/types"; import type { McpServerConfig, McpServerStatus } from "./mcp-client"; import type { RollbackResult, ProjectScripts } from "./agent-tools"; +import type { SandboxCapabilities } from "./command-sandbox"; +import type { TerminalInfo } from "./terminal-manager"; import type { PromptPreset } from "./settings-store"; import type { LocalGgufModel, GpuBackend } from "./llamacpp-manager"; import type { ScheduledTask } from "./scheduled-tasks-store"; @@ -14,6 +16,12 @@ interface ToolExecuteResult { error?: string; } +// Tracked so terminal.close() can remove these deterministically instead of +// relying solely on the main-process exit event to arrive (it normally +// does, but explicit cleanup on close() is cheap insurance against a leaked +// listener if it doesn't). +const terminalListeners = new Map void; exit: (...args: unknown[]) => void }>(); + interface ScreenSourceInfo { id: string; name: string; @@ -96,6 +104,14 @@ contextBridge.exposeInMainWorld("api", { localBackends: { getStatuses: (): Promise => ipcRenderer.invoke("localBackends:getStatuses"), + start: (backend: "mlx" | "rocm" | "vllm", model: string) => ipcRenderer.invoke("localBackends:start", { backend, model }), + stop: (backend: "mlx" | "rocm" | "vllm") => ipcRenderer.invoke("localBackends:stop", backend), + restart: (backend: "mlx" | "rocm" | "vllm", model: string) => ipcRenderer.invoke("localBackends:restart", { backend, model }), + unload: (backend: "mlx" | "rocm" | "vllm") => ipcRenderer.invoke("localBackends:unload", backend), + }, + + pythonRuntimes: { + getStatuses: () => ipcRenderer.invoke("pythonRuntimes:getStatuses"), }, chat: { @@ -161,6 +177,7 @@ contextBridge.exposeInMainWorld("api", { secrets: { has: (key: string) => ipcRenderer.invoke("secrets:has", key), set: (key: string, value: string) => ipcRenderer.invoke("secrets:set", { key, value }), + isEncryptionAvailable: () => ipcRenderer.invoke("secrets:isEncryptionAvailable") as Promise, }, accounts: { @@ -182,6 +199,40 @@ contextBridge.exposeInMainWorld("api", { openLogsFolder: () => ipcRenderer.invoke("app:openLogsFolder"), }, + benchmark: { + run: (request: unknown) => { + const requestId = crypto.randomUUID(); + return { requestId, promise: ipcRenderer.invoke("benchmark:run", { requestId, request }) }; + }, + cancel: (requestId: string) => ipcRenderer.invoke("benchmark:cancel", requestId), + getLast: () => ipcRenderer.invoke("benchmark:getLast"), + exportReport: (result: unknown) => ipcRenderer.invoke("benchmark:exportReport", result), + }, + + energy: { + getDashboard: () => ipcRenderer.invoke("energy:getDashboard"), + clearHistory: () => ipcRenderer.invoke("energy:clearHistory"), + }, + + downloads: { + list: () => ipcRenderer.invoke("downloads:list"), + create: (input: { modelId: string; filename: string; expectedBytes: number; backend?: string }) => ipcRenderer.invoke("downloads:create", input), + pause: (id: string) => ipcRenderer.invoke("downloads:pause", id), + resume: (id: string) => ipcRenderer.invoke("downloads:resume", id), + retry: (id: string) => ipcRenderer.invoke("downloads:retry", id), + cancel: (id: string) => ipcRenderer.invoke("downloads:cancel", id), + delete: (id: string) => ipcRenderer.invoke("downloads:delete", id), + forecast: (id: string) => ipcRenderer.invoke("downloads:forecast", id), + recoveryStatus: () => ipcRenderer.invoke("downloads:recoveryStatus"), + getControls: () => ipcRenderer.invoke("downloads:getControls"), + setControls: (controls: { concurrency: number; bandwidthMbps: number }) => ipcRenderer.invoke("downloads:setControls", controls), + onUpdate: (callback: (jobs: unknown[]) => void) => { + const listener = (_event: unknown, jobs: unknown[]) => callback(jobs); + ipcRenderer.on("downloads:update", listener); + return () => ipcRenderer.removeListener("downloads:update", listener); + }, + }, + menu: { onNewChat: (callback: () => void) => { const listener = () => callback(); @@ -217,9 +268,12 @@ contextBridge.exposeInMainWorld("api", { }, rag: { - indexFiles: (files: AttachedFile[]) => ipcRenderer.invoke("rag:indexFiles", files), - query: (indexId: string, query: string, topK?: number) => - ipcRenderer.invoke("rag:query", { indexId, query, topK }), + indexFolder: (input: { folderPath: string; folderName: string; files: AttachedFile[] }) => + ipcRenderer.invoke("rag:indexFolder", input), + query: (collectionId: string, query: string, topK?: number) => + ipcRenderer.invoke("rag:query", { collectionId, query, topK }), + listCollections: () => ipcRenderer.invoke("rag:listCollections"), + deleteCollection: (id: string) => ipcRenderer.invoke("rag:deleteCollection", id), }, agent: { @@ -230,6 +284,48 @@ contextBridge.exposeInMainWorld("api", { ipcRenderer.invoke("agent:rollbackLastWrite", workspaceRoot), detectScripts: (workspaceRoot: string): Promise => ipcRenderer.invoke("agent:detectScripts", workspaceRoot), + closeWorkspace: (workspaceRoot: string): Promise<{ killedBackgroundTasks: number; killedTerminals: number }> => + ipcRenderer.invoke("agent:closeWorkspace", workspaceRoot), + getSandboxCapabilities: (): Promise => ipcRenderer.invoke("agent:getSandboxCapabilities"), + }, + + // Human-facing interactive terminals (a live pseudo-terminal panel) — + // separate from the model's create_terminal/write_to_terminal/etc. tool + // calls, which poll terminal state through agent.executeTool instead of + // needing a push-streaming channel. Both sides share the same + // terminal-manager.ts session pool, just reached through different paths: + // this one never goes through tool approval, since the human is driving + // it directly. + terminal: { + create: ( + workspaceRoot: string, + opts: { cwd?: string; name?: string }, + onData: (chunk: string) => void, + onExit: (exitCode: number) => void + ): Promise<{ id: string; name: string }> => + ipcRenderer.invoke("terminal:create", { workspaceRoot, opts }).then((created: { id: string; name: string }) => { + const dataListener = (_event: unknown, chunk: string) => onData(chunk); + const exitListener = (_event: unknown, exitCode: number) => onExit(exitCode); + ipcRenderer.on(`terminal:data:${created.id}`, dataListener); + ipcRenderer.on(`terminal:exit:${created.id}`, exitListener); + terminalListeners.set(created.id, { + data: dataListener as (...args: unknown[]) => void, + exit: exitListener as (...args: unknown[]) => void, + }); + return created; + }), + write: (id: string, data: string): Promise => ipcRenderer.invoke("terminal:write", { id, data }), + resize: (id: string, cols: number, rows: number): Promise => ipcRenderer.invoke("terminal:resize", { id, cols, rows }), + close: (id: string): Promise => { + const listeners = terminalListeners.get(id); + if (listeners) { + ipcRenderer.removeListener(`terminal:data:${id}`, listeners.data); + ipcRenderer.removeListener(`terminal:exit:${id}`, listeners.exit); + terminalListeners.delete(id); + } + return ipcRenderer.invoke("terminal:close", id); + }, + list: (workspaceRoot?: string): Promise => ipcRenderer.invoke("terminal:list", workspaceRoot), }, mcp: { diff --git a/app/src/process-tree.test.ts b/app/src/process-tree.test.ts new file mode 100644 index 0000000..b0eed38 --- /dev/null +++ b/app/src/process-tree.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect, vi } from "vitest"; +import { spawn, spawnSync } from "node:child_process"; +import { killProcessTree } from "./process-tree"; + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, spawnSync: vi.fn() }; +}); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +describe("killProcessTree", () => { + // Real process-group kill only makes sense on POSIX — Windows has no + // equivalent to `detached: true` + negative-pid signaling, and Windows CI + // runners don't have `sh`/`sleep` available to build a tree with anyway. + it.skipIf(process.platform === "win32")( + "kills a detached shell and its child process, not just the shell", + async () => { + const child = spawn("sh", ["-c", "sleep 30 & wait"], { detached: true, stdio: "ignore" }); + const shellPid = child.pid!; + // Give the shell a moment to fork `sleep` before we look for it. + await new Promise((r) => setTimeout(r, 200)); + + killProcessTree(shellPid); + await new Promise((r) => setTimeout(r, 200)); + + expect(isAlive(shellPid)).toBe(false); + }, + 10_000 + ); + + it("shells out to taskkill /t /f on win32, regardless of the host platform", () => { + killProcessTree(4321, "SIGTERM", "win32"); + expect(spawnSync).toHaveBeenCalledWith("taskkill", ["/pid", "4321", "/t", "/f"]); + }); + + it("silently no-ops when the target process is already gone", () => { + // A pid essentially guaranteed not to exist. + expect(() => killProcessTree(999_999, "SIGTERM", "linux")).not.toThrow(); + }); +}); diff --git a/app/src/process-tree.ts b/app/src/process-tree.ts new file mode 100644 index 0000000..615830a --- /dev/null +++ b/app/src/process-tree.ts @@ -0,0 +1,31 @@ +import { spawnSync } from "node:child_process"; + +// `spawn(command, {shell: true})` means the tracked pid is the *shell's* +// pid — calling .kill() on it leaves any grandchildren (e.g. `npm run dev` +// spawning a real `node` process) running behind. On POSIX this kills the +// whole process group instead, which requires the child to have been spawned +// with `detached: true` so it becomes its own group leader (see +// startBackgroundCommand in agent-tools.ts). On Windows, `taskkill`'s `/t` +// flag walks the process tree itself, so no special spawn option is needed +// there. +export function killProcessTree( + pid: number, + signal: NodeJS.Signals = "SIGTERM", + platform: NodeJS.Platform = process.platform +): void { + if (platform === "win32") { + spawnSync("taskkill", ["/pid", String(pid), "/t", "/f"]); + return; + } + try { + process.kill(-pid, signal); + } catch { + // Not a group leader (wasn't spawned detached) or already gone — + // fall back to killing just the tracked pid. + try { + process.kill(pid, signal); + } catch { + // Already dead. + } + } +} diff --git a/app/src/providers/types.ts b/app/src/providers/types.ts index accb5f4..4a94a3b 100644 --- a/app/src/providers/types.ts +++ b/app/src/providers/types.ts @@ -41,6 +41,9 @@ export interface ChatMessage { toolName?: string; // User-set bookmark, purely a UI affordance — never sent to a provider. pinned?: boolean; + // Set on the synthetic message the verification loop appends — a UI + // affordance like `pinned`, never sent to a provider. + isVerification?: boolean; } export interface ChatChunk { diff --git a/app/src/python-runtime-manager.ts b/app/src/python-runtime-manager.ts new file mode 100644 index 0000000..62d380f --- /dev/null +++ b/app/src/python-runtime-manager.ts @@ -0,0 +1,218 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { spawn, execFile, type ChildProcess } from "node:child_process"; +import { promisify } from "node:util"; +import { app } from "electron"; +import { killProcessTree } from "./process-tree"; +import { getSpecs, type SystemSpecs } from "./system-specs"; + +const execFileAsync = promisify(execFile); +export const PYTHON_WORKER_PROTOCOL_VERSION = 1; +export type PythonRuntimeFamily = "mlx" | "vllm-cuda" | "vllm-rocm" | "hardware-recommender"; +export type PythonEnvironmentState = "missing" | "healthy" | "drifted" | "incompatible"; + +export interface PythonRuntimeManifest { + family: PythonRuntimeFamily; version: number; python: string; packages: Record; + diskRequirementBytes: number; expectedDownloadBytes: number; protocolVersion: number; + compatibility: string; documentationUrl: string; +} +export interface PythonEnvironmentStatus { + family: PythonRuntimeFamily; state: PythonEnvironmentState; destination: string; pythonPath: string; + pythonVersion: string | null; installedPackages: Record; issues: string[]; + manifest: PythonRuntimeManifest; installCommand: string; repairCommand: string; removeCommand: string; +} + +export const PYTHON_RUNTIME_MANIFESTS: Record = { + mlx: { family: "mlx", version: 1, python: ">=3.10,<3.14", packages: { "mlx-lm": "0.31.3" }, diskRequirementBytes: 2 * 1024 ** 3, expectedDownloadBytes: 900 * 1024 ** 2, protocolVersion: 1, compatibility: "Apple Silicon and macOS", documentationUrl: "https://github.com/ml-explore/mlx-lm" }, + "vllm-cuda": { family: "vllm-cuda", version: 1, python: ">=3.10,<3.15", packages: { vllm: "0.25.1" }, diskRequirementBytes: 10 * 1024 ** 3, expectedDownloadBytes: 4 * 1024 ** 3, protocolVersion: 1, compatibility: "Linux/WSL with a supported NVIDIA CUDA driver", documentationUrl: "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/" }, + "vllm-rocm": { family: "vllm-rocm", version: 1, python: ">=3.10,<3.15", packages: { vllm: "0.25.1" }, diskRequirementBytes: 14 * 1024 ** 3, expectedDownloadBytes: 6 * 1024 ** 3, protocolVersion: 1, compatibility: "Linux/WSL with a supported AMD ROCm stack; source compilation may be required", documentationUrl: "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/" }, + // CPU-only inference for the trained hardware-recommender model (see + // ml/hardware-recommender/) — no GPU/accelerator requirement, runs + // natively on every platform including Windows (unlike vllm-*, which + // needs WSL there), so it's pinned to the CPU-only torch wheel to keep + // the download small. + "hardware-recommender": { family: "hardware-recommender", version: 1, python: ">=3.10,<3.14", packages: { torch: "2.13.0", numpy: "2.2.6" }, diskRequirementBytes: 2 * 1024 ** 3, expectedDownloadBytes: 300 * 1024 ** 2, protocolVersion: 1, compatibility: "Any platform (CPU-only inference)", documentationUrl: "https://github.com/voidstackloop/modelforge/tree/main/ml/hardware-recommender" }, +}; + +function environmentRoot(): string { return path.join(app.getPath("userData"), "python-runtimes"); } +function usesWsl(family: PythonRuntimeFamily): boolean { return process.platform === "win32" && family !== "mlx" && family !== "hardware-recommender"; } +export function environmentDestination(family: PythonRuntimeFamily): string { return usesWsl(family) ? `~/.local/share/modelforge/python-runtimes/${family}` : path.join(environmentRoot(), family); } +export function environmentPython(family: PythonRuntimeFamily, platform: NodeJS.Platform = process.platform): string { + if (platform === "win32" && family !== "mlx") return `${environmentDestination(family)}/bin/python`; + return path.join(environmentDestination(family), platform === "win32" ? "Scripts/python.exe" : "bin/python"); +} +function quote(value: string): string { return process.platform === "win32" ? `"${value.replace(/"/g, '\\"')}"` : `'${value.replace(/'/g, `'"'"'`)}'`; } +function posixQuote(value: string): string { return `'${value.replace(/'/g, `'"'"'`)}'`; } +function packageSpec(manifest: PythonRuntimeManifest): string { return Object.entries(manifest.packages).map(([name, version]) => `${name}==${version}`).join(" "); } +function basePythonCommand(): string { return process.platform === "win32" ? "py -3.12" : "python3.12"; } + +export function compatibilityIssues(family: PythonRuntimeFamily, specs: SystemSpecs): string[] { + if (family === "hardware-recommender") return []; + if (family === "mlx") return specs.platform === "darwin" && specs.arch === "arm64" ? [] : ["MLX requires macOS on Apple Silicon."]; + if (specs.platform === "darwin") return ["vLLM GPU environments require Linux or Windows through WSL."]; + const vendors = new Set(specs.gpus.map((gpu) => gpu.vendor)); + if (family === "vllm-cuda" && !vendors.has("nvidia")) return ["No NVIDIA GPU was detected for the CUDA environment."]; + if (family === "vllm-rocm" && !vendors.has("amd")) return ["No AMD GPU was detected for the ROCm environment."]; + return []; +} + +function pythonVersionSupported(family: PythonRuntimeFamily, version: string): boolean { + const [major, minor] = version.split(".").map(Number); const upperMinor = family === "mlx" || family === "hardware-recommender" ? 14 : 15; + return major === 3 && minor >= 10 && minor < upperMinor; +} + +async function acceleratorIssues(family: PythonRuntimeFamily): Promise { + if (family === "mlx" || family === "hardware-recommender") return []; + try { + if (family === "vllm-cuda") { + await execFileAsync("nvidia-smi", ["--query-gpu=driver_version", "--format=csv,noheader"], { timeout: 5_000, windowsHide: true }); + } else { + await execFileAsync(process.platform === "win32" ? "wsl.exe" : "rocminfo", process.platform === "win32" ? ["--", "rocminfo"] : [], { timeout: 5_000, windowsHide: true, maxBuffer: 1024 * 1024 }); + } + return []; + } catch { + return [family === "vllm-cuda" ? "NVIDIA driver tooling (nvidia-smi) is unavailable." : "ROCm tooling (rocminfo) is unavailable."]; + } +} + +function commands(manifest: PythonRuntimeManifest, destination: string): { installCommand: string; repairCommand: string; removeCommand: string } { + const python = environmentPython(manifest.family); const packages = packageSpec(manifest); + if (usesWsl(manifest.family)) { + const pip = manifest.family === "vllm-rocm" ? `VLLM_TARGET_DEVICE=rocm ${python} -m pip install --no-cache-dir ${packages}` : `${python} -m pip install --only-binary=:all: ${packages}`; + const body = `python3.12 -m venv ${destination} && ${python} -m pip install --upgrade pip==26.1 && ${pip}`; + return { installCommand: `wsl.exe -- bash -lc ${posixQuote(body)}`, repairCommand: `wsl.exe -- bash -lc ${posixQuote(pip.replace("pip install", "pip install --force-reinstall"))}`, removeCommand: `wsl.exe -- rm -rf -- ${destination}` }; + } + // torch's CPU-only wheels live on a separate index — mixed into + // --index-url with the rest of PyPI, pip can't reliably tell "prefer the + // CPU build of torch, but still get numpy from PyPI" (both indexes are + // searched for every package), so torch is installed from the CPU index + // in its own pip invocation and everything else from the default index. + const hardwareRecommenderTarget = (forceReinstall: boolean) => { + const force = forceReinstall ? " --force-reinstall" : ""; + const rest = Object.entries(manifest.packages).filter(([name]) => name !== "torch").map(([name, version]) => `${name}==${version}`).join(" "); + const torchInstall = `${quote(python)} -m pip install --only-binary=:all:${force} --index-url https://download.pytorch.org/whl/cpu torch==${manifest.packages.torch}`; + return rest ? `${torchInstall} && ${quote(python)} -m pip install --only-binary=:all:${force} ${rest}` : torchInstall; + }; + const target = manifest.family === "vllm-rocm" + ? `VLLM_TARGET_DEVICE=rocm ${quote(python)} -m pip install --no-cache-dir ${packages}` + : manifest.family === "hardware-recommender" + ? hardwareRecommenderTarget(false) + : `${quote(python)} -m pip install --only-binary=:all: ${packages}`; + const installCommand = `${basePythonCommand()} -m venv ${quote(destination)} && ${quote(python)} -m pip install --upgrade pip==26.1 && ${target}`; + const repairTarget = manifest.family === "vllm-rocm" + ? target + : manifest.family === "hardware-recommender" + ? hardwareRecommenderTarget(true) + : `${quote(python)} -m pip install --force-reinstall --only-binary=:all: ${packages}`; + const removeCommand = process.platform === "win32" ? `Remove-Item -LiteralPath ${quote(destination)} -Recurse -Force` : `rm -rf -- ${quote(destination)}`; + return { installCommand, repairCommand: repairTarget, removeCommand }; +} + +async function inspectEnvironment(family: PythonRuntimeFamily, specs: SystemSpecs): Promise { + const manifest = PYTHON_RUNTIME_MANIFESTS[family]; const destination = environmentDestination(family); const pythonPath = environmentPython(family); + const hardwareIssues = [...compatibilityIssues(family, specs), ...await acceleratorIssues(family)]; const plan = commands(manifest, destination); + if (hardwareIssues.length) return { family, state: "incompatible", destination, pythonPath, pythonVersion: null, installedPackages: {}, issues: hardwareIssues, manifest, ...plan }; + const exists = usesWsl(family) + ? await execFileAsync("wsl.exe", ["--", "bash", "-lc", `test -x ${pythonPath}`], { timeout: 5_000, windowsHide: true }).then(() => true).catch(() => false) + : fs.existsSync(pythonPath); + if (!exists) return { family, state: "missing", destination, pythonPath, pythonVersion: null, installedPackages: {}, issues: ["Managed virtual environment has not been created."], manifest, ...plan }; + try { + const modules = Object.keys(manifest.packages); + const script = "import json,sys,importlib.metadata as m; names=" + JSON.stringify(modules) + "; print(json.dumps({'python':'.'.join(map(str,sys.version_info[:3])),'packages':{n:m.version(n) for n in names}}))"; + const invocation = usesWsl(family) + ? await execFileAsync("wsl.exe", ["--", "bash", "-lc", `${pythonPath} -c ${quote(script)}`], { timeout: 10_000, windowsHide: true }) + : await execFileAsync(pythonPath, ["-c", script], { timeout: 10_000, windowsHide: true }); + const { stdout } = invocation; + const detected = JSON.parse(stdout) as { python: string; packages: Record }; const issues: string[] = []; + if (!pythonVersionSupported(family, detected.python)) issues.push(`Python ${detected.python} does not satisfy ${manifest.python}.`); + for (const [name, expected] of Object.entries(manifest.packages)) if (detected.packages[name] !== expected) issues.push(`${name} ${detected.packages[name] ?? "missing"}; expected ${expected}.`); + return { family, state: issues.length ? "drifted" : "healthy", destination, pythonPath, pythonVersion: detected.python, installedPackages: detected.packages, issues, manifest, ...plan }; + } catch (error) { + return { family, state: "drifted", destination, pythonPath, pythonVersion: null, installedPackages: {}, issues: [`Environment inspection failed: ${(error as Error).message}`], manifest, ...plan }; + } +} + +export async function getPythonEnvironmentStatuses(): Promise { + const specs = await getSpecs(); return Promise.all((Object.keys(PYTHON_RUNTIME_MANIFESTS) as PythonRuntimeFamily[]).map((family) => inspectEnvironment(family, specs))); +} + +interface WorkerResponse { protocol: number; id: string; ok: boolean; result?: unknown; error?: { code: string; message: string } } +// A request has no inherent deadline otherwise: if the worker process stays +// alive but stops responding (stuck in a long computation, deadlocked, or +// just never writes a line back), the promise in `pending` would never +// settle and would leak forever. +const REQUEST_TIMEOUT_MS = 20_000; +// Bounds how much unterminated stdout `consume` will buffer waiting for a +// newline. A worker that never emits one (buggy print, corrupted output) +// would otherwise grow this string without limit for as long as the process +// lives — this caps the damage and fails loudly instead of leaking memory. +const MAX_BUFFER_BYTES = 8 * 1024 * 1024; +export class ManagedPythonWorker { + private child: ChildProcess | null = null; + private pending = new Map(); + private buffer = ""; + constructor(private readonly family: PythonRuntimeFamily) {} + start(): void { + if (this.child) return; + const python = environmentPython(this.family); + const scriptName = this.family === "hardware-recommender" ? "recommender_worker.py" : "runtime_worker.py"; + const packaged = path.join(process.resourcesPath, "python", scriptName); const script = fs.existsSync(packaged) ? packaged : path.join(__dirname, "..", "python", scriptName); + this.child = spawn(python, [script], { stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32", windowsHide: true }); + this.child.stdout?.setEncoding("utf8"); this.child.stdout?.on("data", (chunk: string) => this.consume(chunk)); + this.child.stderr?.setEncoding("utf8"); this.child.stderr?.on("data", (chunk: string) => { for (const line of chunk.split(/\r?\n/).filter(Boolean)) console.info(`[python-worker:${this.family}] ${line}`); }); + // Without this, a missing/unset-up venv (spawn ENOENT — the common + // case for a family the user hasn't installed yet) emits an 'error' + // event with no listener, which Node treats as an uncaught exception + // and crashes the whole main process instead of just failing this + // one request. + this.child.once("error", (error: Error) => { this.failAllPending(error); this.child = null; }); + this.child.once("exit", () => { this.failAllPending(new Error("Python worker exited")); this.child = null; }); + } + private failAllPending(error: Error): void { + for (const request of this.pending.values()) { clearTimeout(request.timer); request.reject(error); } + this.pending.clear(); + this.buffer = ""; + } + private consume(chunk: string): void { + this.buffer += chunk; + if (this.buffer.length > MAX_BUFFER_BYTES) { + // Something is badly wrong (runaway output, no newlines) — treat + // it as fatal rather than let the process keep growing memory. + const error = new Error(`Python worker "${this.family}" exceeded the ${MAX_BUFFER_BYTES}-byte output buffer without a complete line`); + const child = this.child; + this.failAllPending(error); + if (child?.pid) killProcessTree(child.pid, "SIGKILL"); + this.child = null; + return; + } + const lines = this.buffer.split(/\r?\n/); this.buffer = lines.pop() ?? ""; + for (const line of lines) { + if (!line) continue; + let response: WorkerResponse; + try { + response = JSON.parse(line) as WorkerResponse; + } catch { + console.error(`[python-worker:${this.family}] Ignoring malformed response line: ${line.slice(0, 200)}`); + continue; + } + const pending = this.pending.get(response.id); + if (!pending) continue; + clearTimeout(pending.timer); + this.pending.delete(response.id); + response.ok ? pending.resolve(response.result) : pending.reject(new Error(response.error?.message ?? "Worker request failed")); + } + } + request(method: "health" | "metrics" | "recommend", params: Record = {}): Promise { + this.start(); + const id = crypto.randomUUID(); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Python worker "${this.family}" request "${method}" timed out after ${REQUEST_TIMEOUT_MS}ms`)); + }, REQUEST_TIMEOUT_MS); + this.pending.set(id, { resolve, reject, timer }); + this.child?.stdin?.write(JSON.stringify({ protocol: PYTHON_WORKER_PROTOCOL_VERSION, id, method, params }) + "\n"); + }); + } + async shutdown(): Promise { const child = this.child; if (!child) return; const pid = child.pid; try { const id = crypto.randomUUID(); child.stdin?.write(JSON.stringify({ protocol: PYTHON_WORKER_PROTOCOL_VERSION, id, method: "shutdown", params: {} }) + "\n"); } catch { /* pipe closed */ } await new Promise((resolve) => setTimeout(resolve, 500)); if (pid && this.child) killProcessTree(pid, "SIGTERM"); this.child = null; } +} diff --git a/app/src/rag-db.ts b/app/src/rag-db.ts new file mode 100644 index 0000000..50c35b2 --- /dev/null +++ b/app/src/rag-db.ts @@ -0,0 +1,191 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { app } from "electron"; +import Database from "better-sqlite3"; + +export interface CollectionRow { + id: string; + name: string; + folder_path: string; + embedding_model: string; + created_at: number; + updated_at: number; +} + +export interface DocumentRow { + id: string; + collection_id: string; + path: string; + name: string; + content_hash: string; + size: number; + mtime_ms: number; + page_count: number | null; + indexed_at: number; +} + +export interface ChunkInput { + text: string; + tokenCount: number; + heading: string | null; + page: number | null; + startLine: number; + endLine: number; + embedding: number[]; +} + +export interface ChunkRow { + id: string; + document_id: string; + collection_id: string; + ordinal: number; + text: string; + token_count: number; + heading: string | null; + page: number | null; + start_line: number; + end_line: number; + embedding: Buffer; +} + +function filePath(): string { + return path.join(app.getPath("userData"), "rag.db"); +} + +let db: Database.Database | null = null; + +// Module-level singleton, opened lazily so tests (and any code running +// before app.getPath is available) don't pay for it until first use. +export function getDb(): Database.Database { + if (db) return db; + fs.mkdirSync(path.dirname(filePath()), { recursive: true }); + db = new Database(filePath()); + db.pragma("journal_mode = WAL"); + db.exec(` + CREATE TABLE IF NOT EXISTS collections ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, folder_path TEXT NOT NULL UNIQUE, + embedding_model TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE, + path TEXT NOT NULL, name TEXT NOT NULL, content_hash TEXT NOT NULL, + size INTEGER NOT NULL, mtime_ms INTEGER NOT NULL, page_count INTEGER, indexed_at INTEGER NOT NULL, + UNIQUE(collection_id, path) + ); + CREATE TABLE IF NOT EXISTS chunks ( + id TEXT PRIMARY KEY, document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL, text TEXT NOT NULL, token_count INTEGER NOT NULL, + heading TEXT, page INTEGER, start_line INTEGER NOT NULL, end_line INTEGER NOT NULL, + embedding BLOB NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_chunks_collection ON chunks(collection_id); + CREATE INDEX IF NOT EXISTS idx_documents_collection ON documents(collection_id); + `); + return db; +} + +// Exposed for tests only — the electron mock points app.getPath("userData") +// at a real temp directory shared for the whole test process, so without +// this, rows from one test would leak into the next via the same rag.db file. +export function clearAllForTests(): void { + getDb().exec(`DELETE FROM chunks; DELETE FROM documents; DELETE FROM collections;`); +} + +export function encodeEmbedding(vector: number[]): Buffer { + return Buffer.from(Float32Array.from(vector).buffer); +} + +export function decodeEmbedding(buffer: Buffer): Float32Array { + return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / Float32Array.BYTES_PER_ELEMENT); +} + +export function upsertCollection(input: { id: string; name: string; folderPath: string; embeddingModel: string }): CollectionRow { + const now = Date.now(); + const existing = getCollectionByPath(input.folderPath); + if (existing) { + getDb().prepare(`UPDATE collections SET name = ?, updated_at = ? WHERE id = ?`).run(input.name, now, existing.id); + return { ...existing, name: input.name, updated_at: now }; + } + const row: CollectionRow = { id: input.id, name: input.name, folder_path: input.folderPath, embedding_model: input.embeddingModel, created_at: now, updated_at: now }; + getDb().prepare(`INSERT INTO collections (id, name, folder_path, embedding_model, created_at, updated_at) VALUES (@id, @name, @folder_path, @embedding_model, @created_at, @updated_at)`).run(row); + return row; +} + +export function getCollectionByPath(folderPath: string): CollectionRow | undefined { + return getDb().prepare(`SELECT * FROM collections WHERE folder_path = ?`).get(folderPath) as CollectionRow | undefined; +} + +export function getCollection(id: string): CollectionRow | undefined { + return getDb().prepare(`SELECT * FROM collections WHERE id = ?`).get(id) as CollectionRow | undefined; +} + +export function listCollections(): CollectionRow[] { + return getDb().prepare(`SELECT * FROM collections ORDER BY updated_at DESC`).all() as CollectionRow[]; +} + +export function deleteCollection(id: string): void { + getDb().prepare(`DELETE FROM collections WHERE id = ?`).run(id); +} + +export function touchCollection(id: string): void { + getDb().prepare(`UPDATE collections SET updated_at = ? WHERE id = ?`).run(Date.now(), id); +} + +export function getDocument(collectionId: string, filePath: string): DocumentRow | undefined { + return getDb().prepare(`SELECT * FROM documents WHERE collection_id = ? AND path = ?`).get(collectionId, filePath) as DocumentRow | undefined; +} + +export function listDocuments(collectionId: string): DocumentRow[] { + return getDb().prepare(`SELECT * FROM documents WHERE collection_id = ?`).all(collectionId) as DocumentRow[]; +} + +export function upsertDocument(input: { + id: string; collectionId: string; path: string; name: string; contentHash: string; size: number; mtimeMs: number; pageCount: number | null; +}): void { + const now = Date.now(); + getDb().prepare(` + INSERT INTO documents (id, collection_id, path, name, content_hash, size, mtime_ms, page_count, indexed_at) + VALUES (@id, @collectionId, @path, @name, @contentHash, @size, @mtimeMs, @pageCount, @now) + ON CONFLICT(collection_id, path) DO UPDATE SET + content_hash = excluded.content_hash, size = excluded.size, mtime_ms = excluded.mtime_ms, + page_count = excluded.page_count, indexed_at = excluded.indexed_at + `).run({ ...input, now }); +} + +export function deleteDocument(id: string): void { + getDb().prepare(`DELETE FROM documents WHERE id = ?`).run(id); +} + +export function replaceChunks(documentId: string, collectionId: string, chunks: ChunkInput[]): void { + const db = getDb(); + const del = db.prepare(`DELETE FROM chunks WHERE document_id = ?`); + const insert = db.prepare(` + INSERT INTO chunks (id, document_id, collection_id, ordinal, text, token_count, heading, page, start_line, end_line, embedding) + VALUES (@id, @document_id, @collection_id, @ordinal, @text, @token_count, @heading, @page, @start_line, @end_line, @embedding) + `); + const tx = db.transaction((rows: ChunkInput[]) => { + del.run(documentId); + rows.forEach((chunk, ordinal) => { + insert.run({ + id: `${documentId}:${ordinal}`, document_id: documentId, collection_id: collectionId, ordinal, + text: chunk.text, token_count: chunk.tokenCount, heading: chunk.heading, page: chunk.page, + start_line: chunk.startLine, end_line: chunk.endLine, embedding: encodeEmbedding(chunk.embedding), + }); + }); + }); + tx(chunks); +} + +export function chunksForCollection(collectionId: string): (ChunkRow & { doc_path: string; doc_name: string })[] { + return getDb().prepare(` + SELECT chunks.*, documents.path AS doc_path, documents.name AS doc_name + FROM chunks JOIN documents ON documents.id = chunks.document_id + WHERE chunks.collection_id = ? + `).all(collectionId) as (ChunkRow & { doc_path: string; doc_name: string })[]; +} + +export function countChunks(collectionId: string): number { + const row = getDb().prepare(`SELECT COUNT(*) AS n FROM chunks WHERE collection_id = ?`).get(collectionId) as { n: number }; + return row.n; +} diff --git a/app/src/rag.test.ts b/app/src/rag.test.ts index 7f8f47a..98773d5 100644 --- a/app/src/rag.test.ts +++ b/app/src/rag.test.ts @@ -1,29 +1,64 @@ -import { describe, it, expect } from "vitest"; -import { chunkText, cosineSimilarity } from "./rag"; +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { chunkDocument, cosineSimilarity, indexFolder, query, listCollections, deleteCollection } from "./rag"; +import { clearAllForTests, listDocuments, getCollectionByPath } from "./rag-db"; +import type { AttachedFile } from "./file-reader"; -describe("chunkText", () => { - it("returns the whole text as one chunk when it's under the chunk size", () => { - expect(chunkText("short text")).toEqual(["short text"]); - }); +// Deterministic fake embeddings keyed by exact prompt text, so retrieval +// ranking is verifiable without a real embedding model. Falls back to a +// neutral vector for anything not explicitly registered below. +const VECTORS: Record = {}; +function mockEmbeddingFor(text: string, vector: number[]) { + VECTORS[text] = vector; +} + +beforeEach(() => { + clearAllForTests(); + for (const key of Object.keys(VECTORS)) delete VECTORS[key]; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: { body: string }) => { + const body = JSON.parse(init.body); + return { ok: true, json: async () => ({ embedding: VECTORS[body.prompt] ?? [0, 0, 1] }) }; + }) + ); +}); - it("splits long text into overlapping chunks", () => { - const text = "a".repeat(2500); - const chunks = chunkText(text); +function file(partial: Partial): AttachedFile { + return { name: "file.txt", path: "/test/file.txt", content: "content", truncated: false, ...partial }; +} - expect(chunks.length).toBeGreaterThan(1); - // every chunk is at most CHUNK_SIZE (1000) characters +describe("chunkDocument", () => { + it("returns the whole text as one chunk when it's under the token budget", () => { + const chunks = chunkDocument("line one\nline two\nline three"); + expect(chunks).toHaveLength(1); + expect(chunks[0].startLine).toBe(1); + expect(chunks[0].endLine).toBe(3); + }); + + it("never splits a line across chunk boundaries", () => { + const longLine = "word ".repeat(2000); // ~2500 tokens on one physical line + const content = `${longLine}\nshort line after`; + const chunks = chunkDocument(content); for (const chunk of chunks) { - expect(chunk.length).toBeLessThanOrEqual(1000); + for (const line of chunk.text.split("\n")) { + expect(content.includes(line)).toBe(true); + } } }); - it("covers the entire input with no gaps between chunks", () => { - const text = Array.from({ length: 2500 }, (_, i) => String(i % 10)).join(""); - const chunks = chunkText(text); - // reconstruct coverage by checking every character position appears - // in at least one chunk at the expected offset (accounting for overlap) - const reconstructed = chunks[0] + chunks.slice(1).map((c) => c.slice(150)).join(""); - expect(reconstructed).toBe(text); + it("splits long content into multiple token-budgeted chunks with overlap", () => { + const lines = Array.from({ length: 400 }, (_, i) => `line number ${i} with some extra words to add tokens`); + const chunks = chunkDocument(lines.join("\n")); + expect(chunks.length).toBeGreaterThan(1); + // consecutive chunks overlap: the second chunk's start line is not + // simply one past the first chunk's end line. + expect(chunks[1].startLine).toBeLessThanOrEqual(chunks[0].endLine); + }); + + it("tags each chunk with the nearest preceding markdown heading", () => { + const content = "# Intro\nsome intro text\n\n## Details\nmore text here"; + const chunks = chunkDocument(content); + expect(chunks[0].heading).toBe("Intro"); }); }); @@ -31,16 +66,115 @@ describe("cosineSimilarity", () => { it("returns 1 for identical vectors", () => { expect(cosineSimilarity([1, 2, 3], [1, 2, 3])).toBeCloseTo(1); }); + it("returns 0 for a zero vector instead of NaN", () => { + expect(cosineSimilarity([0, 0], [1, 2])).toBe(0); + }); +}); - it("returns 0 for orthogonal vectors", () => { - expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0); +describe("indexFolder", () => { + it("creates a persistent collection with chunk/document counts", async () => { + mockEmbeddingFor("hello world", [1, 0, 0]); + const result = await indexFolder({ + folderPath: "/test/folder-a", folderName: "folder-a", + files: [file({ path: "/test/folder-a/a.txt", name: "a.txt", content: "hello world" })], + }); + expect(result.embedded).toBe(true); + expect(result.documentCount).toBe(1); + expect(result.chunkCount).toBe(1); + expect(getCollectionByPath("/test/folder-a")).toBeDefined(); }); - it("returns -1 for opposite vectors", () => { - expect(cosineSimilarity([1, 2], [-1, -2])).toBeCloseTo(-1); + it("skips re-embedding a file whose content hash is unchanged", async () => { + mockEmbeddingFor("hello world", [1, 0, 0]); + const files = [file({ path: "/test/folder-b/a.txt", name: "a.txt", content: "hello world" })]; + await indexFolder({ folderPath: "/test/folder-b", folderName: "folder-b", files }); + + const fetchSpy = global.fetch as unknown as ReturnType; + const callsBefore = fetchSpy.mock.calls.length; + const result = await indexFolder({ folderPath: "/test/folder-b", folderName: "folder-b", files }); + expect(fetchSpy.mock.calls.length).toBe(callsBefore); // no new embed calls + expect(result.documentCount).toBe(1); }); - it("returns 0 for a zero vector instead of NaN", () => { - expect(cosineSimilarity([0, 0], [1, 2])).toBe(0); + it("re-embeds a file whose content changed", async () => { + mockEmbeddingFor("version one", [1, 0, 0]); + mockEmbeddingFor("version two", [0, 1, 0]); + const folderPath = "/test/folder-c"; + await indexFolder({ folderPath, folderName: "folder-c", files: [file({ path: `${folderPath}/a.txt`, content: "version one" })] }); + + const fetchSpy = global.fetch as unknown as ReturnType; + const callsBefore = fetchSpy.mock.calls.length; + await indexFolder({ folderPath, folderName: "folder-c", files: [file({ path: `${folderPath}/a.txt`, content: "version two" })] }); + expect(fetchSpy.mock.calls.length).toBeGreaterThan(callsBefore); + }); + + it("removes documents that are no longer present on re-index", async () => { + mockEmbeddingFor("keep me", [1, 0, 0]); + mockEmbeddingFor("remove me", [0, 1, 0]); + const folderPath = "/test/folder-d"; + const first = await indexFolder({ + folderPath, folderName: "folder-d", + files: [file({ path: `${folderPath}/keep.txt`, content: "keep me" }), file({ path: `${folderPath}/gone.txt`, content: "remove me" })], + }); + expect(listDocuments(first.collectionId)).toHaveLength(2); + + const second = await indexFolder({ + folderPath, folderName: "folder-d", + files: [file({ path: `${folderPath}/keep.txt`, content: "keep me" })], + }); + const docs = listDocuments(second.collectionId); + expect(docs).toHaveLength(1); + expect(docs[0].path).toBe(`${folderPath}/keep.txt`); + }); + + it("reports a failure reason instead of silently swallowing an embedding failure", async () => { + vi.stubGlobal("fetch", vi.fn(async () => ({ ok: false }))); + const result = await indexFolder({ + folderPath: "/test/folder-e", folderName: "folder-e", + files: [file({ path: "/test/folder-e/a.txt", content: "hello" })], + }); + expect(result.embedded).toBe(false); + expect(result.error).toMatch(/unavailable/i); + }); +}); + +describe("query", () => { + it("returns results sorted by similarity with full metadata", async () => { + mockEmbeddingFor("about cats", [1, 0, 0]); + mockEmbeddingFor("about dogs", [0, 1, 0]); + mockEmbeddingFor("cats please", [0.9, 0.1, 0]); + const folderPath = "/test/folder-f"; + const indexed = await indexFolder({ + folderPath, folderName: "folder-f", + files: [ + file({ path: `${folderPath}/cats.txt`, name: "cats.txt", content: "about cats" }), + file({ path: `${folderPath}/dogs.txt`, name: "dogs.txt", content: "about dogs" }), + ], + }); + + const results = await query(indexed.collectionId, "cats please", 8); + expect(results).toHaveLength(2); + expect(results[0].source.name).toBe("cats.txt"); + expect(results[0].score).toBeGreaterThan(results[1].score); + expect(results[0].startLine).toBe(1); + expect(results[0].endLine).toBe(1); + }); + + it("returns an empty array for an unknown collection", async () => { + expect(await query("does-not-exist", "anything")).toEqual([]); + }); +}); + +describe("listCollections / deleteCollection", () => { + it("lists persisted collections and removes them on delete", async () => { + mockEmbeddingFor("hello", [1, 0, 0]); + const result = await indexFolder({ + folderPath: "/test/folder-g", folderName: "folder-g", + files: [file({ path: "/test/folder-g/a.txt", content: "hello" })], + }); + expect(listCollections().some((c) => c.collectionId === result.collectionId)).toBe(true); + + deleteCollection(result.collectionId); + expect(listCollections().some((c) => c.collectionId === result.collectionId)).toBe(false); }); }); diff --git a/app/src/rag.ts b/app/src/rag.ts index f0cde35..3fdc5f8 100644 --- a/app/src/rag.ts +++ b/app/src/rag.ts @@ -1,53 +1,83 @@ +import * as path from "node:path"; +import { createHash } from "node:crypto"; import { randomUUID } from "node:crypto"; import * as ollama from "./ollama-manager"; -import type { AttachedFile } from "./file-reader"; +import * as media from "./media"; +import { approximateTokens } from "./benchmark-runner"; +import { findPdfFiles, type AttachedFile } from "./file-reader"; +import * as ragDb from "./rag-db"; +import type { ChunkRow } from "./rag-db"; -export interface RagChunk { +export const DEFAULT_EMBEDDING_MODEL = "nomic-embed-text"; +const EMBED_BATCH_SIZE = 8; +const TARGET_TOKENS = 400; +const OVERLAP_TOKENS = 60; +const HEADING_RE = /^#{1,6}\s+(.*)/; + +export interface LineChunk { text: string; - source: string; + startLine: number; + endLine: number; + heading: string | null; + tokenCount: number; } -interface IndexedChunk extends RagChunk { - embedding: number[]; -} +// Document-aware, token-budgeted chunking: packs consecutive lines into a +// chunk until ~TARGET_TOKENS is hit (never splitting mid-line), carries the +// nearest preceding markdown heading as context, and re-includes ~OVERLAP_TOKENS +// worth of trailing lines at the start of the next chunk so a fact split +// across a chunk boundary still appears whole in at least one chunk. +export function chunkDocument(content: string): LineChunk[] { + const lines = content.split(/\r?\n/); + const chunks: LineChunk[] = []; + let currentHeading: string | null = null; + let buffer: { idx: number; line: string; heading: string | null }[] = []; + let bufferTokens = 0; -interface RagIndexEntry { - chunks: IndexedChunk[]; - createdAt: number; -} + const flush = () => { + if (buffer.length === 0) return; + const text = buffer.map((b) => b.line).join("\n"); + if (text.trim().length > 0) { + chunks.push({ + text, + startLine: buffer[0].idx + 1, + endLine: buffer[buffer.length - 1].idx + 1, + heading: buffer[0].heading, + tokenCount: approximateTokens(text), + }); + } + }; -export interface IndexFilesResult { - indexId: string; - chunkCount: number; - embedded: boolean; -} + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const headingMatch = line.match(HEADING_RE); + if (headingMatch) currentHeading = headingMatch[1].trim(); -const indexes = new Map(); -const MAX_INDEXES = 8; -const MAX_CHUNKS = 200; -const CHUNK_SIZE = 1000; -const CHUNK_OVERLAP = 150; -const EMBED_BATCH_SIZE = 8; -const EMBEDDING_MODEL = "nomic-embed-text"; - -export function chunkText(text: string): string[] { - if (text.length <= CHUNK_SIZE) return [text]; - const chunks: string[] = []; - let start = 0; - while (start < text.length) { - const end = Math.min(start + CHUNK_SIZE, text.length); - chunks.push(text.slice(start, end)); - start += CHUNK_SIZE - CHUNK_OVERLAP; + const lineTokens = approximateTokens(line); + if (buffer.length > 0 && bufferTokens + lineTokens > TARGET_TOKENS) { + flush(); + let overlapTokens = 0; + const overlap: typeof buffer = []; + for (let j = buffer.length - 1; j >= 0 && overlapTokens < OVERLAP_TOKENS; j--) { + overlap.unshift(buffer[j]); + overlapTokens += approximateTokens(buffer[j].line); + } + buffer = overlap; + bufferTokens = overlapTokens; + } + buffer.push({ idx: i, line, heading: currentHeading }); + bufferTokens += lineTokens; } + flush(); return chunks; } -async function embed(text: string): Promise { +async function embed(text: string, model: string): Promise { try { const res = await fetch(`${ollama.getHost()}/api/embeddings`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model: EMBEDDING_MODEL, prompt: text }), + body: JSON.stringify({ model, prompt: text }), signal: AbortSignal.timeout(15000), }); if (!res.ok) return null; @@ -58,7 +88,20 @@ async function embed(text: string): Promise { } } -export function cosineSimilarity(a: number[], b: number[]): number { +async function embedChunks(texts: string[], model: string): Promise { + const out: number[][] = []; + for (let i = 0; i < texts.length; i += EMBED_BATCH_SIZE) { + const batch = texts.slice(i, i + EMBED_BATCH_SIZE); + const embeddings = await Promise.all(batch.map((t) => embed(t, model))); + for (const e of embeddings) { + if (!e) return null; + out.push(e); + } + } + return out; +} + +export function cosineSimilarity(a: number[] | Float32Array, b: number[] | Float32Array): number { let dot = 0; let normA = 0; let normB = 0; @@ -71,58 +114,158 @@ export function cosineSimilarity(a: number[], b: number[]): number { return dot / (Math.sqrt(normA) * Math.sqrt(normB)); } -function evictOldestIfFull(): void { - if (indexes.size < MAX_INDEXES) return; - let oldestId: string | null = null; - let oldestTime = Infinity; - for (const [id, entry] of indexes) { - if (entry.createdAt < oldestTime) { - oldestTime = entry.createdAt; - oldestId = id; - } - } - if (oldestId) indexes.delete(oldestId); +function hashContent(content: string): string { + return createHash("sha256").update(content).digest("hex"); } -export async function indexFiles(files: AttachedFile[]): Promise { - const rawChunks: RagChunk[] = []; - for (const file of files) { - for (const piece of chunkText(file.content)) { - rawChunks.push({ text: piece, source: file.name }); - if (rawChunks.length >= MAX_CHUNKS) break; +export interface IndexFolderInput { + folderPath: string; + folderName: string; + files: AttachedFile[]; + embeddingModel?: string; +} + +export interface CollectionSummary { + collectionId: string; + name: string; + folderPath?: string; + documentCount: number; + chunkCount: number; + embeddingModel: string; + updatedAt?: number; + embedded: boolean; + error?: string; +} + +// Indexes (or incrementally re-indexes) a folder into a persistent, +// named collection. Unchanged files (by content hash) are skipped entirely — +// only new/changed files get re-chunked and re-embedded. Files present in the +// DB for this collection but absent from the current file list are removed +// (stale-document cleanup on manual re-index; this is not live file watching). +export async function indexFolder(input: IndexFolderInput): Promise { + const embeddingModel = input.embeddingModel ?? DEFAULT_EMBEDDING_MODEL; + const existingCollection = ragDb.getCollectionByPath(input.folderPath); + const collection = ragDb.upsertCollection({ + id: existingCollection?.id ?? randomUUID(), + name: input.folderName, + folderPath: input.folderPath, + embeddingModel: existingCollection?.embedding_model ?? embeddingModel, + }); + + const currentPaths = new Set(); + let embedFailure: string | null = null; + + for (const file of input.files) { + currentPaths.add(file.path); + if (embedFailure) break; + const hash = hashContent(file.content); + const existingDoc = ragDb.getDocument(collection.id, file.path); + if (existingDoc && existingDoc.content_hash === hash) continue; + + const chunks = chunkDocument(file.content); + const embeddings = chunks.length > 0 ? await embedChunks(chunks.map((c) => c.text), collection.embedding_model) : []; + if (!embeddings) { embedFailure = `Embedding model "${collection.embedding_model}" is unavailable`; break; } + + const documentId = existingDoc?.id ?? randomUUID(); + ragDb.upsertDocument({ + id: documentId, collectionId: collection.id, path: file.path, name: file.name, + contentHash: hash, size: file.content.length, mtimeMs: Date.now(), pageCount: null, + }); + ragDb.replaceChunks(documentId, collection.id, chunks.map((c, i) => ({ + text: c.text, tokenCount: c.tokenCount, heading: c.heading, page: null, + startLine: c.startLine, endLine: c.endLine, embedding: embeddings[i], + }))); + } + + if (!embedFailure) { + for (const pdfPath of findPdfFiles(input.folderPath)) { + currentPaths.add(pdfPath); + if (embedFailure) break; + const { text, pages } = await media.extractPdfPages(pdfPath); + const hash = hashContent(text); + const existingDoc = ragDb.getDocument(collection.id, pdfPath); + if (existingDoc && existingDoc.content_hash === hash) continue; + + const pageChunks: (LineChunk & { page: number })[] = []; + for (const page of pages) { + for (const c of chunkDocument(page.text)) pageChunks.push({ ...c, page: page.num }); + } + const embeddings = pageChunks.length > 0 ? await embedChunks(pageChunks.map((c) => c.text), collection.embedding_model) : []; + if (!embeddings) { embedFailure = `Embedding model "${collection.embedding_model}" is unavailable`; break; } + + const name = path.relative(input.folderPath, pdfPath).split(path.sep).join("/"); + const documentId = existingDoc?.id ?? randomUUID(); + ragDb.upsertDocument({ + id: documentId, collectionId: collection.id, path: pdfPath, name, + contentHash: hash, size: text.length, mtimeMs: Date.now(), pageCount: pages.length, + }); + ragDb.replaceChunks(documentId, collection.id, pageChunks.map((c, i) => ({ + text: c.text, tokenCount: c.tokenCount, heading: c.heading, page: c.page, + startLine: c.startLine, endLine: c.endLine, embedding: embeddings[i], + }))); } - if (rawChunks.length >= MAX_CHUNKS) break; } - const indexed: IndexedChunk[] = []; - for (let i = 0; i < rawChunks.length; i += EMBED_BATCH_SIZE) { - const batch = rawChunks.slice(i, i + EMBED_BATCH_SIZE); - const embeddings = await Promise.all(batch.map((c) => embed(c.text))); - for (let j = 0; j < batch.length; j++) { - const embedding = embeddings[j]; - // Embedding model unavailable (not pulled, or Ollama unreachable) — - // bail out entirely so the caller falls back to dumping raw content. - if (!embedding) return { indexId: "", chunkCount: 0, embedded: false }; - indexed.push({ ...batch[j], embedding }); + if (!embedFailure) { + for (const doc of ragDb.listDocuments(collection.id)) { + if (!currentPaths.has(doc.path)) ragDb.deleteDocument(doc.id); } } - evictOldestIfFull(); - const indexId = randomUUID(); - indexes.set(indexId, { chunks: indexed, createdAt: Date.now() }); - return { indexId, chunkCount: indexed.length, embedded: true }; + ragDb.touchCollection(collection.id); + const documentCount = ragDb.listDocuments(collection.id).length; + const chunkCount = ragDb.countChunks(collection.id); + + return { + collectionId: collection.id, name: collection.name, folderPath: collection.folder_path, + documentCount, chunkCount, embeddingModel: collection.embedding_model, + embedded: !embedFailure, error: embedFailure ?? undefined, + }; +} + +export interface RagResult { + text: string; + score: number; + source: { path: string; name: string }; + heading: string | null; + page: number | null; + startLine: number; + endLine: number; +} + +function toResult(row: ChunkRow & { doc_path: string; doc_name: string }, score: number): RagResult { + return { + text: row.text, score, source: { path: row.doc_path, name: row.doc_name }, + heading: row.heading, page: row.page, startLine: row.start_line, endLine: row.end_line, + }; } -export async function query(indexId: string, queryText: string, topK = 8): Promise { - const entry = indexes.get(indexId); - if (!entry) return []; +// Retrieval is still brute-force cosine similarity over every chunk in the +// collection — fine at the scale a single folder attach produces, but a real +// ANN index is the natural next step for large collections (deferred). +export async function query(collectionId: string, queryText: string, topK = 8): Promise { + const collection = ragDb.getCollection(collectionId); + if (!collection) return []; + const rows = ragDb.chunksForCollection(collectionId); - const queryEmbedding = await embed(queryText); + const queryEmbedding = await embed(queryText, collection.embedding_model); if (!queryEmbedding) { - return entry.chunks.slice(0, topK).map(({ text, source }) => ({ text, source })); + return rows.slice(0, topK).map((row) => toResult(row, 0)); } - const scored = entry.chunks.map((c) => ({ ...c, score: cosineSimilarity(c.embedding, queryEmbedding) })); + const scored = rows.map((row) => ({ row, score: cosineSimilarity(ragDb.decodeEmbedding(row.embedding), queryEmbedding) })); scored.sort((a, b) => b.score - a.score); - return scored.slice(0, topK).map(({ text, source }) => ({ text, source })); + return scored.slice(0, topK).map(({ row, score }) => toResult(row, score)); +} + +export function listCollections(): CollectionSummary[] { + return ragDb.listCollections().map((c) => ({ + collectionId: c.id, name: c.name, folderPath: c.folder_path, + documentCount: ragDb.listDocuments(c.id).length, chunkCount: ragDb.countChunks(c.id), + embeddingModel: c.embedding_model, updatedAt: c.updated_at, embedded: true, + })); +} + +export function deleteCollection(id: string): void { + ragDb.deleteCollection(id); } diff --git a/app/src/resource-monitor.test.ts b/app/src/resource-monitor.test.ts new file mode 100644 index 0000000..344a42f --- /dev/null +++ b/app/src/resource-monitor.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { monitorProcess } from "./resource-monitor"; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +// pidusage's per-call latency varies a lot by platform (Windows CI runners +// in particular can be slow/jittery, e.g. when it shells out for process +// stats) — polling for the expected outcome instead of sleeping a fixed +// duration avoids flaking on a runner where one polling interval legitimately +// takes longer than the fixed wait this used to use. +async function waitFor(predicate: () => boolean, timeoutMs = 5000): Promise { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > timeoutMs) throw new Error("timed out waiting for condition"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +describe("monitorProcess", () => { + let child: ChildProcess | undefined; + let stop: (() => void) | undefined; + + afterEach(() => { + stop?.(); + if (child?.pid && isAlive(child.pid)) child.kill(); + }); + + it( + "kills the process and reports why when it exceeds the memory limit", + async () => { + child = spawn("node", ["-e", "setTimeout(() => {}, 10000)"], { stdio: "ignore" }); + const pid = child.pid!; + const reasons: string[] = []; + + // Any real node process uses well over 1MB RSS, so this fires on the + // very first poll. + stop = monitorProcess(pid, { maxMemoryMB: 1 }, (reason) => reasons.push(reason), 50); + + // Windows CI runners can take noticeably longer than Vitest's + // default 5000ms per-test timeout for pidusage's first sample + // (it shells out for process stats there) — 5000ms was both this + // waitFor's internal budget and Vitest's outer one, so a slow + // first poll hit the outer timeout before waitFor's own, more + // descriptive error could ever fire. Both are widened here, + // consistently, so the intended error surfaces instead. + await waitFor(() => reasons.length > 0, 15_000); + + expect(reasons).toHaveLength(1); + expect(reasons[0]).toMatch(/exceeded the 1MB memory limit/); + expect(isAlive(pid)).toBe(false); + }, + 20_000 + ); + + it("does nothing when no limits are configured", async () => { + child = spawn("node", ["-e", "setTimeout(() => {}, 10000)"], { stdio: "ignore" }); + const pid = child.pid!; + const reasons: string[] = []; + + stop = monitorProcess(pid, {}, (reason) => reasons.push(reason), 50); + await new Promise((r) => setTimeout(r, 200)); + + expect(reasons).toHaveLength(0); + expect(isAlive(pid)).toBe(true); + }); + + it("stops cleanly without throwing once the watched process has already exited", async () => { + child = spawn("node", ["-e", "process.exit(0)"], { stdio: "ignore" }); + const pid = child.pid!; + const reasons: string[] = []; + + stop = monitorProcess(pid, { maxMemoryMB: 100_000 }, (reason) => reasons.push(reason), 50); + await new Promise((r) => setTimeout(r, 300)); + + expect(reasons).toHaveLength(0); + }); + + it("stop() prevents any further callback even if a poll was already in flight", async () => { + child = spawn("node", ["-e", "setTimeout(() => {}, 10000)"], { stdio: "ignore" }); + const pid = child.pid!; + const reasons: string[] = []; + + stop = monitorProcess(pid, { maxMemoryMB: 1 }, (reason) => reasons.push(reason), 50); + stop(); + await new Promise((r) => setTimeout(r, 300)); + + expect(reasons).toHaveLength(0); + expect(isAlive(pid)).toBe(true); + }); +}); diff --git a/app/src/resource-monitor.ts b/app/src/resource-monitor.ts new file mode 100644 index 0000000..a9ba1c3 --- /dev/null +++ b/app/src/resource-monitor.ts @@ -0,0 +1,75 @@ +import pidusage from "pidusage"; +import { killProcessTree } from "./process-tree"; + +export interface ResourceLimits { + maxMemoryMB?: number; + maxCpuPercent?: number; +} + +const DEFAULT_POLL_INTERVAL_MS = 2000; + +// A safety net against a runaway process (a build loop stuck consuming +// memory, a background dev server that leaks), not a real resource quota +// system: pidusage measures the single tracked pid's own usage, not a full +// process-tree/cgroup total, so a process that does its heavy lifting in a +// child won't be caught by the *measurement* here — though killProcessTree +// below still takes the whole tree down once a breach is detected on the +// pid that is being watched. +// +// Returns a function that stops monitoring — always call it once the +// process being watched has exited on its own, otherwise the interval +// leaks. `intervalMs` is exposed for tests; production code should leave it +// at the default. +export function monitorProcess( + pid: number, + limits: ResourceLimits, + onExceeded: (reason: string) => void, + intervalMs = DEFAULT_POLL_INTERVAL_MS +): () => void { + if (!limits.maxMemoryMB && !limits.maxCpuPercent) return () => {}; + + let stopped = false; + const stop = () => { + if (stopped) return; + stopped = true; + clearInterval(timer); + }; + + const timer = setInterval(async () => { + if (stopped) return; + // Not pre-declared with an explicit type: `ReturnType` only sees pidusage's *last* overload (the array-of-pids + // one) since TS's ReturnType utility can't pick from an overload set + // the way a real call site does — annotating this variable's type + // that way would silently widen it to the wrong shape. Left to + // call-site inference instead, which correctly resolves the + // single-pid `Promise` overload. + let memoryMB: number; + let cpuPercent: number; + try { + const stats = await pidusage(pid); + memoryMB = stats.memory / (1024 * 1024); + cpuPercent = stats.cpu; + } catch { + // Process already exited — nothing left to monitor. + stop(); + return; + } + if (stopped) return; // stopped while the async pidusage() call was in flight + + if (limits.maxMemoryMB && memoryMB > limits.maxMemoryMB) { + stop(); + killProcessTree(pid); + onExceeded(`exceeded the ${limits.maxMemoryMB}MB memory limit (was using ${memoryMB.toFixed(0)}MB)`); + return; + } + if (limits.maxCpuPercent && cpuPercent > limits.maxCpuPercent) { + stop(); + killProcessTree(pid); + onExceeded(`exceeded the ${limits.maxCpuPercent}% CPU limit (was using ${cpuPercent.toFixed(0)}%)`); + } + }, intervalMs); + timer.unref?.(); + + return stop; +} diff --git a/app/src/secrets-store.ts b/app/src/secrets-store.ts index 779d1a7..97e10ff 100644 --- a/app/src/secrets-store.ts +++ b/app/src/secrets-store.ts @@ -1,6 +1,7 @@ import * as path from "node:path"; import { app, safeStorage } from "electron"; import { readJson, writeJson } from "./json-store"; +import { logger } from "./logger"; function filePath(): string { return path.join(app.getPath("userData"), "secrets.json"); @@ -14,6 +15,19 @@ function writeAll(data: Record): void { writeJson(filePath(), data); } +// Exposed so callers (and Settings UI) can warn the user before a key ends +// up unencrypted, rather than only finding out after the fact. +export function isEncryptionAvailable(): boolean { + return safeStorage.isEncryptionAvailable(); +} + +// Environments without an OS credential store (e.g. some Linux setups with no +// keyring) can't use safeStorage at all. We still allow storing the key there +// rather than silently dropping it or hard-blocking the feature, but this is +// no longer a silent fallback: it's logged at warn level (surfaced in Settings +// -> Diagnostics -> Copy diagnostic info) and the Settings UI checks +// isEncryptionAvailable() up front to show a plaintext-storage warning next to +// every API key field instead of the normal "encrypted at rest" note. export function setSecret(key: string, value: string): void { const all = readAll(); if (!value) { @@ -21,8 +35,7 @@ export function setSecret(key: string, value: string): void { } else if (safeStorage.isEncryptionAvailable()) { all[key] = safeStorage.encryptString(value).toString("base64"); } else { - // Fallback for environments without an OS credential store (e.g. some - // Linux setups with no keyring). Better to work than to silently drop the key. + logger.warn(`No OS credential store available; storing secret "${key}" unencrypted in secrets.json`); all[key] = value; } writeAll(all); diff --git a/app/src/settings-store.ts b/app/src/settings-store.ts index bf1aa75..9c9bd32 100644 --- a/app/src/settings-store.ts +++ b/app/src/settings-store.ts @@ -2,6 +2,7 @@ import * as path from "node:path"; import { app } from "electron"; import { readJson, writeJson } from "./json-store"; import type { McpServerConfig } from "./mcp-client"; +import type { TimeOfUseTariff } from "./energy-types"; export interface PromptVersion { prompt: string; @@ -73,6 +74,15 @@ export interface AppSettings { // directory) since the two backends use incompatible on-disk layouts. llamaCppModelsDir?: string; llamaCppGpuBackend?: "auto" | "vulkan" | "cuda" | "metal" | "cpu"; + // Embedding model used to index new RAG collections. Existing collections + // keep whatever model they were created with (stored per-collection in + // rag.db) — this only governs newly created ones. + ragEmbeddingModel?: string; + // Which backend runs a model. "automatic" (the default) picks per-model + // based on file format and detected hardware — see resolveAutomaticRuntime + // in system-specs.ts. Any other value pins every recommendation/download + // to that one backend regardless of format/hardware fit. + preferredRuntime?: "automatic" | "ollama" | "llamacpp" | "vllm" | "mlx"; // User-added OpenAI-compatible endpoints (Groq, Mistral, DeepSeek, xAI, // OpenRouter, or anything else that speaks the same API) — each one's // API key is stored separately via secretsStore, keyed by its id. @@ -99,6 +109,42 @@ export interface AppSettings { // vLLM runtime. The `vllm` executable is discovered from PATH by default. vllmModels?: string[]; vllmCommand?: string; + // Agent-mode sandboxing. Network tools (web_search, fetch_url, + // http_request, capture_page_screenshot, the GitHub tools) are gated by + // this flag directly — 100% enforceable on every platform, since it's + // just refusing to run the tool at all rather than trying to block + // network access after the fact. + networkToolsEnabled?: boolean; + // Safety-net resource caps applied to run_command/run_code/background + // commands (see resource-monitor.ts) — generous defaults, meant to catch + // a runaway process rather than act as a real resource quota system. + sandboxMaxMemoryMB?: number; + sandboxMaxCpuPercent?: number; + // Verification loop (Agent mode): once a turn ends with the model + // calling no more tools, optionally run these command(s) for real and + // feed the result back before treating the turn as actually finished. + // Off by default — running commands automatically after every turn + // would surprise anyone who didn't explicitly turn it on. + // verificationCommands unset falls back to the workspace's detected + // build/test scripts (see detectProjectScripts) at the point of use. + verificationEnabled?: boolean; + verificationCommands?: string[]; + // Distinct from agentMaxSteps — bounds verify-fail-retry cycles + // specifically, so a persistently failing check can't loop forever. + verificationMaxRetries?: number; + energyMonitoringEnabled?: boolean; + electricityPricePerKwh?: number; + energyCurrency?: string; + timeOfUseTariffs?: TimeOfUseTariff[]; + manualCpuWatts?: number; + manualGpuWatts?: number; + manualSystemIdleWatts?: number; + includeIdleSystemConsumption?: boolean; + energyUsageRetentionDays?: number; + energySampleIntervalSeconds?: number; + downloadGlobalConcurrency?: number; + downloadBandwidthMbps?: number; + gridIntensityGCo2PerKwh?: number; } const DEFAULTS: AppSettings = { @@ -118,6 +164,19 @@ const DEFAULTS: AppSettings = { reduceMotion: false, agentMaxSteps: 25, llamaCppMaxCachedModels: 2, + networkToolsEnabled: true, + sandboxMaxMemoryMB: 2048, + verificationEnabled: false, + verificationMaxRetries: 3, + energyMonitoringEnabled: false, + electricityPricePerKwh: 0.2, + energyCurrency: "USD", + timeOfUseTariffs: [], + includeIdleSystemConsumption: true, + energyUsageRetentionDays: 365, + energySampleIntervalSeconds: 2, + downloadGlobalConcurrency: 2, + downloadBandwidthMbps: 0, }; function filePath(): string { diff --git a/app/src/system-specs.test.ts b/app/src/system-specs.test.ts index 4016d20..0e047bc 100644 --- a/app/src/system-specs.test.ts +++ b/app/src/system-specs.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { classifyGpuVendor, recommendModels, type SystemSpecs } from "./system-specs"; +import { classifyGpuVendor, detectModelFormat, recommendModels, recommendModelsWithML, resolveAutomaticRuntime, type SystemSpecs } from "./system-specs"; function baseSpecs(overrides: Partial = {}): SystemSpecs { return { @@ -12,6 +12,11 @@ function baseSpecs(overrides: Partial = {}): SystemSpecs { gpu: null, gpus: [], totalVramGB: null, + largestGpuVramGB: null, + gpuInterconnect: "none", + tensorParallelSupported: false, + cpuMemoryBandwidthGBps: 50, + cpuMemoryBandwidthMeasured: false, ...overrides, }; } @@ -48,7 +53,7 @@ describe("recommendModels", () => { it("falls back to RAM-based sizing when there's no GPU", () => { const result = recommendModels(baseSpecs({ totalRAMGB: 16 })); expect(result.usableVRAMGB).toBe(0); - expect(result.usableRAMGB).toBeCloseTo(11.2); + expect(result.usableRAMGB).toBeCloseTo(6); }); it("sums VRAM across multiple GPUs instead of using only the first one", () => { @@ -61,20 +66,24 @@ describe("recommendModels", () => { totalVramGB: 48, }); const result = recommendModels(specs); - // 48GB total * 0.9 headroom factor - expect(result.usableVRAMGB).toBeCloseTo(43.2); + // 48GB aggregate remains visible, but is not treated as one pool. + // aggregateUsableVramGB is rounded to 1 decimal place (same precision + // as every other GB figure this function reports), so the raw + // 21.12 * 2 = 42.24 product displays as 42.2, not 42.24. + expect(result.usableVRAMGB).toBeCloseTo(42.2, 1); }); - it("uses whichever of RAM or VRAM gives more usable headroom", () => { + it("does not treat aggregate VRAM as one GPU", () => { const specs = baseSpecs({ - totalRAMGB: 8, - gpu: { name: "Big GPU", vramGB: 80, vendor: "nvidia" }, - gpus: [{ name: "Big GPU", vramGB: 80, vendor: "nvidia" }], - totalVramGB: 80, + totalRAMGB: 16, freeRAMGB: 12, + gpu: { name: "GPU 0", vramGB: 12, vendor: "nvidia" }, + gpus: [{ name: "GPU 0", vramGB: 12, vendor: "nvidia" }, { name: "GPU 1", vramGB: 12, vendor: "nvidia" }], + totalVramGB: 24, largestGpuVramGB: 12, gpuInterconnect: "pcie", tensorParallelSupported: false, }); const result = recommendModels(specs); - expect(result.effectiveGB).toBeCloseTo(result.usableVRAMGB); - expect(result.usableVRAMGB).toBeGreaterThan(result.usableRAMGB); + expect(result.largestUsableGpuGB).toBeCloseTo(10.6, 1); + expect(result.aggregateUsableVramGB).toBeCloseTo(21.1, 1); + expect(result.models.find((model) => model.name === "qwen2.5-coder:14b")?.outcome).not.toBe("Runs fully on GPU"); }); it("marks a model as runsOnGpu only when it fits within usable VRAM", () => { @@ -89,7 +98,86 @@ describe("recommendModels", () => { const large = result.models.find((m) => m.name === "llama3.1:70b")!; expect(tiny.runsOnGpu).toBe(true); expect(large.runsOnGpu).toBe(false); - // Still fits overall thanks to the large RAM pool, just not on the GPU. - expect(large.fits).toBe(true); + expect(["Runs with partial offload", "CPU-only but usable", "Likely out of memory"]).toContain(large.outcome); }); + + it("requires tensor parallelism only when per-GPU shards and interconnect support fit", () => { + const result = recommendModels(baseSpecs({ totalRAMGB: 16, freeRAMGB: 10, + gpu: { name: "RTX 6000 Ada", vramGB: 48, vendor: "nvidia" }, + gpus: [{ name: "RTX 6000 Ada", vramGB: 48, vendor: "nvidia" }, { name: "RTX 6000 Ada", vramGB: 48, vendor: "nvidia" }], + totalVramGB: 96, largestGpuVramGB: 48, gpuInterconnect: "nvlink", tensorParallelSupported: true, + }), { quantization: "Q4_K_M", contextLength: 4096, runtime: "vllm" }); + expect(result.models.find((model) => model.name === "llama3.1:70b")?.outcome).toBe("Requires tensor parallelism"); + }); + + it("includes quantization, KV cache, overhead, offload, and measured speed", () => { + const result = recommendModels(baseSpecs(), { quantization: "Q8_0", contextLength: 32768 }, [{ model: "llama3.2:3b", tokensPerSecond: 17.5 }]); + const model = result.models.find((item) => item.name === "llama3.2:3b")!; + expect(model.estimatedKvCacheGB).toBeGreaterThan(0); + expect(model.runtimeOverheadGB).toBeGreaterThan(0); + expect(model.estimatedWeightGB).toBeGreaterThan(3); + expect(model.measuredTokensPerSecond).toBe(17.5); + }); +}); + +describe("detectModelFormat", () => { + it("identifies GGUF and safetensors files by extension", () => { + expect(detectModelFormat("llama-3.2-3b.Q4_K_M.gguf")).toBe("gguf"); + expect(detectModelFormat("model-00001-of-00002.safetensors")).toBe("safetensors"); + expect(detectModelFormat("model.safetensors.index.json")).toBe("safetensors"); + }); + it("identifies MLX repos by the mlx-community publisher convention", () => { + expect(detectModelFormat("mlx-community/Llama-3.2-3B-Instruct-4bit")).toBe("mlx"); + expect(detectModelFormat("some-org/Qwen2.5-7B-mlx-4bit")).toBe("mlx"); + }); + it("falls back to unknown for anything unrecognized", () => { + expect(detectModelFormat("meta-llama/Llama-3.1-8B-Instruct")).toBe("unknown"); + }); +}); + +describe("resolveAutomaticRuntime", () => { + const linuxNvidia = { platform: "linux" as const, arch: "x64", gpus: [{ name: "RTX 4090", vramGB: 24, vendor: "nvidia" as const }] }; + const linuxAmd = { platform: "linux" as const, arch: "x64", gpus: [{ name: "Radeon RX 7900", vramGB: 24, vendor: "amd" as const }] }; + const linuxNoGpu = { platform: "linux" as const, arch: "x64", gpus: [] }; + const appleSilicon = { platform: "darwin" as const, arch: "arm64", gpus: [{ name: "Apple M3 Max", vramGB: null, vendor: "apple" as const }] }; + const intelMac = { platform: "darwin" as const, arch: "x64", gpus: [{ name: "Intel Iris", vramGB: null, vendor: "intel" as const }] }; + + it("routes GGUF to llama.cpp regardless of vendor", () => { + expect(resolveAutomaticRuntime("gguf", linuxNvidia)).toBe("llamacpp"); + expect(resolveAutomaticRuntime("gguf", linuxAmd)).toBe("llamacpp"); + expect(resolveAutomaticRuntime("gguf", appleSilicon)).toBe("llamacpp"); + expect(resolveAutomaticRuntime("gguf", linuxNoGpu)).toBe("llamacpp"); + }); + it("routes safetensors to vLLM only on NVIDIA/ROCm hardware", () => { + expect(resolveAutomaticRuntime("safetensors", linuxNvidia)).toBe("vllm"); + expect(resolveAutomaticRuntime("safetensors", linuxAmd)).toBe("vllm"); + expect(resolveAutomaticRuntime("safetensors", linuxNoGpu)).toBe("transformers"); + }); + it("routes MLX-format models to MLX only on Apple Silicon", () => { + expect(resolveAutomaticRuntime("mlx", appleSilicon)).toBe("mlx"); + expect(resolveAutomaticRuntime("mlx", intelMac)).toBe("transformers"); + expect(resolveAutomaticRuntime("mlx", linuxNvidia)).toBe("transformers"); + }); + it("routes Ollama-library models to Ollama, or MLX on Apple Silicon", () => { + expect(resolveAutomaticRuntime("ollama", linuxNvidia)).toBe("ollama"); + expect(resolveAutomaticRuntime("ollama", appleSilicon)).toBe("mlx"); + }); + it("falls back to Transformers for unrecognized formats", () => { + expect(resolveAutomaticRuntime("unknown", linuxNvidia)).toBe("transformers"); + }); +}); + +describe("recommendModelsWithML", () => { + // The managed Python worker isn't installed in the test environment (no + // venv under the mocked userData dir), so every model's ML prediction + // rejects and recommendModelsWithML must fall back to the pure heuristic + // result rather than throwing or hanging. + it("falls back to the heuristic recommendModels() output when the ML worker is unavailable", async () => { + const specs = baseSpecs({ totalRAMGB: 16 }); + const heuristic = recommendModels(specs); + const withMl = await recommendModelsWithML(specs); + expect(withMl.models.map((m) => m.name)).toEqual(heuristic.models.map((m) => m.name)); + expect(withMl.models.map((m) => m.recommendedRuntime)).toEqual(heuristic.models.map((m) => m.recommendedRuntime)); + expect(withMl.best).toBe(heuristic.best); + }, 15_000); }); diff --git a/app/src/system-specs.ts b/app/src/system-specs.ts index 230cb9a..3336cb7 100644 --- a/app/src/system-specs.ts +++ b/app/src/system-specs.ts @@ -1,5 +1,6 @@ import * as os from "node:os"; import { execFile } from "node:child_process"; +import { ManagedPythonWorker } from "./python-runtime-manager"; export interface GpuInfo { name: string; @@ -20,6 +21,11 @@ export interface SystemSpecs { gpu: GpuInfo | null; gpus: GpuInfo[]; totalVramGB: number | null; + largestGpuVramGB: number | null; + gpuInterconnect: "nvlink" | "pcie" | "unified" | "none" | "unknown"; + tensorParallelSupported: boolean; + cpuMemoryBandwidthGBps: number; + cpuMemoryBandwidthMeasured: boolean; } export interface ModelCatalogEntry { @@ -38,12 +44,31 @@ export interface RecommendedModel extends ModelCatalogEntry { fits: boolean; runsOnGpu: boolean; recommended: boolean; + outcome: RecommendationOutcome; + quantization: string; + estimatedWeightGB: number; + estimatedKvCacheGB: number; + runtimeOverheadGB: number; + totalRequiredGB: number; + expectedGpuOffloadPercent: number; + estimatedTokensPerSecond: number; + measuredTokensPerSecond?: number; + reason: string; + recommendedRuntime: "ollama" | "llamacpp" | "vllm" | "mlx"; } +export type RecommendationOutcome = "Runs fully on GPU" | "Runs with partial offload" | "CPU-only but usable" | "Requires tensor parallelism" | "Likely out of memory"; +export const QUANTIZATION_BITS = { Q2_K: 2.625, Q3_K_M: 3.5, Q4_K_M: 4.75, Q5_K_M: 5.5, Q6_K: 6.56, Q8_0: 8.5 } as const; +export interface RecommendationOptions { quantization?: keyof typeof QUANTIZATION_BITS; contextLength?: number; runtime?: "automatic" | "ollama" | "llamacpp" | "vllm" | "mlx" } +export interface BenchmarkObservation { model: string; tokensPerSecond: number; promptTokensPerSecond?: number; timeToFirstTokenMs?: number } + export interface ModelRecommendations { usableRAMGB: number; usableVRAMGB: number; - effectiveGB: number; + largestUsableGpuGB: number; + aggregateUsableVramGB: number; + cpuMemoryBandwidthGBps: number; + gpuInterconnect: SystemSpecs["gpuInterconnect"]; best: string | null; models: RecommendedModel[]; } @@ -83,6 +108,46 @@ export function classifyGpuVendor(name: string): GpuVendor { return "unknown"; } +// The five backends the app can actually load a model with. +export type ConcreteRuntime = "ollama" | "llamacpp" | "vllm" | "mlx" | "transformers"; + +// A model's on-disk/distribution format, inferred from a filename or HF repo +// id — this is what "Automatic" branches on, alongside detected hardware. +export type ModelFormat = "gguf" | "safetensors" | "mlx" | "ollama" | "unknown"; + +// Infers format from a filename or Hugging Face repo id. GGUF and +// safetensors are identified by file extension; MLX has no extension of its +// own (it's a directory of .safetensors plus an mlx-flavored config.json), +// so it's identified by the `mlx-community/` publisher convention that +// mlx-lm's own conversion tooling uses. +export function detectModelFormat(identifier: string): ModelFormat { + const lower = identifier.toLowerCase(); + if (lower.endsWith(".gguf")) return "gguf"; + if (lower.endsWith(".safetensors") || lower.endsWith(".safetensors.index.json")) return "safetensors"; + if (/(^|\/)mlx-community\//.test(lower) || /-mlx(-|$)/.test(lower)) return "mlx"; + return "unknown"; +} + +// The policy behind the "Automatic" runtime: given a model's format and this +// machine's hardware, pick the concrete backend to run it with. +// GGUF -> llama.cpp (runs on any vendor, or CPU-only) +// Safetensors + NVIDIA/AMD (ROCm) -> vLLM +// MLX-format model + Apple Silicon -> MLX +// Ollama-library model -> Ollama (MLX on Apple Silicon, which +// has the edge over llama.cpp there) +// Anything else (safetensors with no supported GPU, MLX format off Apple +// Silicon, an unrecognized format) -> Transformers, the universal fallback +// for architectures the other backends can't load. +export function resolveAutomaticRuntime(format: ModelFormat, specs: Pick): ConcreteRuntime { + const isAppleSilicon = specs.platform === "darwin" && specs.arch === "arm64"; + const hasVllmGpu = specs.gpus.some((gpu) => gpu.vendor === "nvidia" || gpu.vendor === "amd"); + if (format === "ollama") return isAppleSilicon ? "mlx" : "ollama"; + if (format === "gguf") return "llamacpp"; + if (format === "mlx" && isAppleSilicon) return "mlx"; + if (format === "safetensors" && hasVllmGpu) return "vllm"; + return "transformers"; +} + function execFileP(cmd: string, args: string[]): Promise { return new Promise((resolve) => { execFile(cmd, args, { timeout: 3000 }, (err, stdout) => { @@ -175,44 +240,205 @@ async function detectGpus(): Promise { return []; } +async function detectGpuInterconnect(gpus: GpuInfo[]): Promise { + if (gpus.length < 2) return gpus.some((gpu) => gpu.vendor === "apple") ? "unified" : "none"; + if (gpus.every((gpu) => gpu.vendor === "apple")) return "unified"; + if (gpus.every((gpu) => gpu.vendor === "nvidia")) { + const topology = await execFileP("nvidia-smi", ["topo", "-m"]); + return topology && /NV\d+|NVLink/i.test(topology) ? "nvlink" : "pcie"; + } + return "unknown"; +} + +function estimateCpuMemoryBandwidthGBps(cpuModel: string, cores: number, platform: NodeJS.Platform): number { + if (platform === "darwin" && /Apple M[1-9]/i.test(cpuModel)) return /Ultra/i.test(cpuModel) ? 600 : /Max/i.test(cpuModel) ? 400 : /Pro/i.test(cpuModel) ? 200 : 100; + if (/EPYC|Xeon.*Max/i.test(cpuModel)) return Math.min(350, Math.max(100, cores * 4)); + if (/Threadripper|Xeon/i.test(cpuModel)) return Math.min(220, Math.max(70, cores * 3)); + return Math.min(120, Math.max(25, cores * 4)); +} + export async function getSpecs(): Promise { const cpus = os.cpus() || []; const gpus = await detectGpus(); const knownVram = gpus.map((g) => g.vramGB).filter((v): v is number => v !== null); const totalVramGB = knownVram.length > 0 ? +knownVram.reduce((a, b) => a + b, 0).toFixed(1) : null; + const largestGpuVramGB = knownVram.length > 0 ? Math.max(...knownVram) : null; + const gpuInterconnect = await detectGpuInterconnect(gpus); + const cpuModel = cpus[0] ? cpus[0].model : "Unknown CPU"; return { totalRAMGB: +(os.totalmem() / 1e9).toFixed(1), freeRAMGB: +(os.freemem() / 1e9).toFixed(1), - cpuModel: cpus[0] ? cpus[0].model : "Unknown CPU", + cpuModel, cpuCores: cpus.length, platform: os.platform(), arch: os.arch(), gpu: gpus[0] ?? null, gpus, totalVramGB, + largestGpuVramGB, + gpuInterconnect, + tensorParallelSupported: gpus.length > 1 && gpus.every((gpu) => gpu.vendor === "nvidia") && gpuInterconnect !== "unknown", + cpuMemoryBandwidthGBps: estimateCpuMemoryBandwidthGBps(cpuModel, cpus.length, os.platform()), + cpuMemoryBandwidthMeasured: false, }; } -export function recommendModels(specs: SystemSpecs): ModelRecommendations { - // Leave headroom for the OS and the app itself on both RAM and VRAM. - const usableRAMGB = +(specs.totalRAMGB * 0.7).toFixed(1); - const usableVRAMGB = specs.totalVramGB ? +(specs.totalVramGB * 0.9).toFixed(1) : 0; - const effectiveGB = Math.max(usableRAMGB, usableVRAMGB); +const PARAMETER_OVERRIDES: Record = { "phi3.5": 3.8, "mistral-nemo": 12, "gemma2:9b": 9, "devstral-small": 24, "qwen3:30b-a3b": 30, "command-r-plus": 104 }; + +function modelParametersB(model: ModelCatalogEntry): number { + return PARAMETER_OVERRIDES[model.name] ?? Number(model.name.match(/(\d+(?:\.\d+)?)b/i)?.[1] ?? model.minRAMGB); +} +function runtimeOverhead(weightGB: number, runtime: NonNullable): number { + const base = runtime === "vllm" ? 1.6 : runtime === "mlx" ? 0.9 : 0.7; + return base + weightGB * (runtime === "vllm" ? 0.12 : 0.08); +} +function observationFor(model: string, history: BenchmarkObservation[]): BenchmarkObservation | undefined { + const normalized = model.toLowerCase().split(":")[0]; + return [...history].reverse().find((item) => item.model.toLowerCase().includes(normalized) || normalized.includes(item.model.toLowerCase().split(":")[0])); +} - const fitting = MODEL_CATALOG.filter((m) => m.minRAMGB <= effectiveGB); - const best = fitting.sort((a, b) => b.minRAMGB - a.minRAMGB)[0]; +export function recommendModels(specs: SystemSpecs, options: RecommendationOptions = {}, history: BenchmarkObservation[] = []): ModelRecommendations { + const quantization = options.quantization ?? "Q4_K_M"; const bits = QUANTIZATION_BITS[quantization]; + const contextLength = Math.max(2_048, Math.min(131_072, options.contextLength ?? 8_192)); + // "automatic" (the default) resolves via resolveAutomaticRuntime, which for + // this Ollama-library catalog only ever yields "mlx" or "ollama" — hence + // the narrowing cast. + const selectedRuntime = options.runtime && options.runtime !== "automatic" ? options.runtime : (resolveAutomaticRuntime("ollama", specs) as "ollama" | "mlx"); + const usableRAMGB = +Math.max(0, Math.min(specs.totalRAMGB * 0.72, specs.freeRAMGB > 2 ? specs.freeRAMGB - 2 : specs.totalRAMGB * 0.55)).toFixed(1); + const knownGpuMemory = specs.gpus.map((gpu) => gpu.vramGB).filter((value): value is number => value !== null); + const largestUsableGpuGB = +(Math.max(0, specs.largestGpuVramGB ?? (knownGpuMemory.length ? Math.max(...knownGpuMemory) : 0)) * 0.88).toFixed(1); + const aggregateUsableVramGB = +(knownGpuMemory.reduce((sum, value) => sum + value * 0.88, 0)).toFixed(1); + const smallestUsableGpuGB = knownGpuMemory.length ? Math.min(...knownGpuMemory) * 0.88 : 0; - return { - usableRAMGB, - usableVRAMGB, - effectiveGB: +effectiveGB.toFixed(1), - best: best ? best.name : null, - models: MODEL_CATALOG.map((m) => ({ - ...m, - fits: m.minRAMGB <= effectiveGB, - runsOnGpu: usableVRAMGB > 0 && m.minRAMGB <= usableVRAMGB, - recommended: best ? m.name === best.name : false, - })), - }; + const models = MODEL_CATALOG.map((model) => { + const parametersB = modelParametersB(model); + const estimatedWeightGB = parametersB * bits / 8 * 1.12 + 0.35; + const estimatedKvCacheGB = Math.max(0.12, 0.17 * Math.sqrt(parametersB / 7)) * contextLength / 1024; + const overhead = runtimeOverhead(estimatedWeightGB, selectedRuntime); + const totalRequiredGB = estimatedWeightGB + estimatedKvCacheGB + overhead; + const gpuBudgetForWeights = Math.max(0, largestUsableGpuGB - estimatedKvCacheGB - overhead); + const expectedGpuOffloadPercent = Math.max(0, Math.min(100, gpuBudgetForWeights / estimatedWeightGB * 100)); + const cpuRemainingGB = estimatedWeightGB * (1 - expectedGpuOffloadPercent / 100) + Math.min(overhead, 1.2); + const fullGpu = totalRequiredGB <= largestUsableGpuGB; + const tensorParallel = !fullGpu && specs.tensorParallelSupported && knownGpuMemory.length > 1 && + estimatedWeightGB / knownGpuMemory.length + estimatedKvCacheGB + overhead <= smallestUsableGpuGB && totalRequiredGB <= aggregateUsableVramGB; + const partial = !fullGpu && !tensorParallel && expectedGpuOffloadPercent >= 5 && cpuRemainingGB <= usableRAMGB; + const cpuOnly = !fullGpu && !tensorParallel && !partial && totalRequiredGB <= usableRAMGB; + let outcome: RecommendationOutcome; let reason: string; let recommendedRuntime = selectedRuntime; + if (fullGpu) { outcome = "Runs fully on GPU"; reason = `Weights, ${contextLength.toLocaleString()}-token KV cache, and runtime overhead fit the largest GPU.`; } + else if (tensorParallel) { outcome = "Requires tensor parallelism"; recommendedRuntime = "vllm"; reason = `Does not fit one GPU; estimated shards fit ${knownGpuMemory.length} GPUs using ${specs.gpuInterconnect}.`; } + else if (partial) { outcome = "Runs with partial offload"; recommendedRuntime = "llamacpp"; reason = `${expectedGpuOffloadPercent.toFixed(0)}% GPU offload estimated; remaining weights fit usable system RAM.`; } + else if (cpuOnly) { outcome = "CPU-only but usable"; recommendedRuntime = "llamacpp"; reason = `Fits usable RAM; speed is constrained by approximately ${specs.cpuMemoryBandwidthGBps} GB/s CPU memory bandwidth.`; } + else { outcome = "Likely out of memory"; reason = `Needs approximately ${totalRequiredGB.toFixed(1)} GB after KV cache and runtime overhead; neither GPU nor RAM/offload layouts fit safely.`; } + const cpuTps = specs.cpuMemoryBandwidthGBps / Math.max(estimatedWeightGB, 0.5) * 0.65; + const gpuTps = 42 * Math.sqrt(7 / Math.max(parametersB, 0.5)) * (4.75 / bits); + const estimatedTokensPerSecond = outcome === "Likely out of memory" ? 0 : outcome === "Runs fully on GPU" || outcome === "Requires tensor parallelism" ? gpuTps : outcome === "Runs with partial offload" ? cpuTps + (gpuTps - cpuTps) * expectedGpuOffloadPercent / 100 : cpuTps; + const measured = observationFor(model.name, history); + return { ...model, fits: outcome !== "Likely out of memory", runsOnGpu: fullGpu, recommended: false, outcome, quantization, + estimatedWeightGB: +estimatedWeightGB.toFixed(2), estimatedKvCacheGB: +estimatedKvCacheGB.toFixed(2), runtimeOverheadGB: +overhead.toFixed(2), + totalRequiredGB: +totalRequiredGB.toFixed(2), expectedGpuOffloadPercent: +expectedGpuOffloadPercent.toFixed(0), + estimatedTokensPerSecond: +estimatedTokensPerSecond.toFixed(1), measuredTokensPerSecond: measured?.tokensPerSecond, reason, recommendedRuntime }; + }); + const best = [...models].filter((model) => model.fits).sort((a, b) => modelParametersB(b) - modelParametersB(a))[0]; + if (best) best.recommended = true; + return { usableRAMGB, usableVRAMGB: aggregateUsableVramGB, largestUsableGpuGB, aggregateUsableVramGB, + cpuMemoryBandwidthGBps: specs.cpuMemoryBandwidthGBps, gpuInterconnect: specs.gpuInterconnect, best: best?.name ?? null, models }; +} + +// Mixture-of-experts models need extra expert-memory accounting the trained +// recommender doesn't have leaderboard metadata for (see ml/hardware-recommender +// README) — flagging them lets the model at least apply its is_moe adjustment. +const MOE_MODELS = new Set(["qwen3:30b-a3b"]); + +function mlPlatform(platform: NodeJS.Platform): "windows" | "linux" | "macos" { + return platform === "win32" ? "windows" : platform === "darwin" ? "macos" : "linux"; +} + +// Mirrors the vendor -> backend convention used elsewhere in this app +// (frontend/src/lib/gpu.ts's recommendGpuBackend): AMD goes through ROCm on +// Linux (native driver support) but Vulkan on Windows (no ROCm llama.cpp +// prebuilds there), Intel always through Vulkan. +function mlGpuBackend(specs: SystemSpecs): "none" | "cuda" | "rocm" | "metal" | "vulkan" | "directml" { + const vendors = new Set(specs.gpus.map((gpu) => gpu.vendor)); + if (vendors.has("apple")) return "metal"; + if (vendors.has("nvidia")) return "cuda"; + if (vendors.has("amd")) return specs.platform === "linux" ? "rocm" : "vulkan"; + if (vendors.has("intel")) return "vulkan"; + return "none"; +} + +const ML_RUNTIME_MAP: Record = { + "llama.cpp-cpu": "llamacpp", "llama.cpp-cuda": "llamacpp", "llama.cpp-rocm": "llamacpp", + "llama.cpp-metal": "llamacpp", "llama.cpp-vulkan": "llamacpp", mlx: "mlx", vllm: "vllm", +}; + +const ML_OUTCOME_MAP: Record = { + "Runs comfortably": "Runs fully on GPU", + "May require partial GPU offload": "Runs with partial offload", + "CPU only": "CPU-only but usable", + "Insufficient memory": "Likely out of memory", +}; + +interface MlRecommendation { + quantization: string; + fit: string; + runtime: string; + estimatedContextTokens: number; + estimatedTokensPerSecond: number; +} + +let recommenderWorker: ManagedPythonWorker | null = null; +function getRecommenderWorker(): ManagedPythonWorker { + if (!recommenderWorker) recommenderWorker = new ManagedPythonWorker("hardware-recommender"); + return recommenderWorker; +} + +async function predictWithML(model: ModelCatalogEntry, specs: SystemSpecs): Promise { + try { + const result = await getRecommenderWorker().request("recommend", { + model_params_b: modelParametersB(model), + quality_score: 30, + is_moe: MOE_MODELS.has(model.name), + ram_gb: specs.totalRAMGB, + vram_gb: specs.largestGpuVramGB ?? 0, + cpu_cores: specs.cpuCores, + platform: mlPlatform(specs.platform), + gpu_backend: mlGpuBackend(specs), + }); + return result as MlRecommendation; + } catch { + // Worker not installed, or install unhealthy/drifted — caller keeps + // the heuristic result for this model rather than failing the whole + // recommendation request over one optional dependency. + return null; + } +} + +// Enhances recommendModels()'s heuristic output with predictions from the +// trained hardware-recommender model (ml/hardware-recommender/) wherever the +// managed Python worker is available, falling back to the pure heuristic per +// model otherwise — the ML worker being absent (not yet installed) or briefly +// unhealthy never blocks a recommendation from being returned. +export async function recommendModelsWithML(specs: SystemSpecs, options: RecommendationOptions = {}, history: BenchmarkObservation[] = []): Promise { + const baseline = recommendModels(specs, options, history); + const predictions = await Promise.all(baseline.models.map((model) => predictWithML(model, specs))); + + const models = baseline.models.map((model, index) => { + const prediction = predictions[index]; + if (!prediction) return model; + const recommendedRuntime = ML_RUNTIME_MAP[prediction.runtime] ?? model.recommendedRuntime; + const outcome = ML_OUTCOME_MAP[prediction.fit] ?? model.outcome; + return { + ...model, + recommendedRuntime, + outcome, + fits: outcome !== "Likely out of memory", + estimatedTokensPerSecond: prediction.estimatedTokensPerSecond, + reason: `ML-predicted: ${prediction.fit.toLowerCase()} at ${prediction.quantization}, ~${prediction.estimatedContextTokens.toLocaleString()}-token context.`, + }; + }); + const best = [...models].filter((model) => model.fits).sort((a, b) => modelParametersB(b) - modelParametersB(a))[0]; + for (const model of models) model.recommended = best ? model.name === best.name : false; + return { ...baseline, models, best: best?.name ?? null }; } diff --git a/app/src/terminal-manager.test.ts b/app/src/terminal-manager.test.ts new file mode 100644 index 0000000..0a69a8a --- /dev/null +++ b/app/src/terminal-manager.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { + createTerminal, + writeToTerminal, + resizeTerminal, + readTerminalOutput, + closeTerminal, + closeAllForWorkspace, + closeAll, + listTerminals, +} from "./terminal-manager"; + +// 5000ms was tight enough that a slow PTY spawn on Windows CI could still +// hit vitest's outer testTimeout (see app/vitest.config.ts) before this +// helper's own, more descriptive error had a chance to fire. +function waitFor(predicate: () => boolean, timeoutMs = 12_000): Promise { + const start = Date.now(); + return new Promise((resolve, reject) => { + const check = () => { + if (predicate()) return resolve(); + if (Date.now() - start > timeoutMs) return reject(new Error("timed out waiting for condition")); + setTimeout(check, 20); + }; + check(); + }); +} + +describe("terminal-manager", () => { + let workspace: string; + + beforeEach(() => { + workspace = fs.mkdtempSync(path.join(os.tmpdir(), "terminal-manager-test-")); + }); + + afterEach(() => { + closeAll(); + }); + + it("creates a terminal and reflects a typed command's output in scrollback", async () => { + const { id } = createTerminal(workspace, {}, () => {}, () => {}); + writeToTerminal(id, "echo hello-from-pty\r"); + await waitFor(() => readTerminalOutput(id).includes("hello-from-pty")); + expect(readTerminalOutput(id)).toContain("hello-from-pty"); + }); + + it("streams data to the onData callback as it arrives, not just into scrollback", async () => { + let received = ""; + const { id } = createTerminal(workspace, {}, (chunk) => (received += chunk), () => {}); + writeToTerminal(id, "echo streamed-chunk\r"); + await waitFor(() => received.includes("streamed-chunk")); + expect(received).toContain("streamed-chunk"); + void id; + }); + + it("reports exit via onExit and marks the terminal as not alive", async () => { + let exitCode: number | null = null; + const { id } = createTerminal(workspace, {}, () => {}, (code) => (exitCode = code)); + writeToTerminal(id, "exit 0\r"); + await waitFor(() => exitCode !== null); + expect(listTerminals(workspace).find((t) => t.id === id)?.alive).toBe(false); + }); + + it("closes and forgets only the terminals belonging to the given workspace", () => { + const otherWorkspace = fs.mkdtempSync(path.join(os.tmpdir(), "terminal-manager-test-other-")); + createTerminal(workspace, {}, () => {}, () => {}); + const b = createTerminal(otherWorkspace, {}, () => {}, () => {}); + + const closedCount = closeAllForWorkspace(workspace); + + expect(closedCount).toBe(1); + expect(listTerminals(workspace)).toEqual([]); + expect(listTerminals(otherWorkspace).map((t) => t.id)).toEqual([b.id]); + closeTerminal(b.id); + fs.rmSync(otherWorkspace, { recursive: true, force: true }); + }); + + it("throws for an unknown terminal id", () => { + expect(() => writeToTerminal("does-not-exist", "x")).toThrow(/No terminal/); + expect(() => readTerminalOutput("does-not-exist")).toThrow(/No terminal/); + expect(() => resizeTerminal("does-not-exist", 80, 24)).toThrow(/No terminal/); + }); + + it("caps the number of concurrently open terminals", () => { + for (let i = 0; i < 5; i++) createTerminal(workspace, { name: `t-${i}` }, () => {}, () => {}); + expect(() => createTerminal(workspace, {}, () => {}, () => {})).toThrow(/Already have/); + }); + + it("treats closing an already-closed terminal as a harmless no-op", () => { + const { id } = createTerminal(workspace, {}, () => {}, () => {}); + closeTerminal(id); + expect(() => closeTerminal(id)).not.toThrow(); + expect(listTerminals(workspace)).toEqual([]); + }); +}); diff --git a/app/src/terminal-manager.ts b/app/src/terminal-manager.ts new file mode 100644 index 0000000..e78c19d --- /dev/null +++ b/app/src/terminal-manager.ts @@ -0,0 +1,143 @@ +import * as path from "node:path"; +import * as pty from "node-pty"; +import type { IPty } from "node-pty"; +import { wrapCommand } from "./command-sandbox"; +import { killProcessTree } from "./process-tree"; +import { resolveSafePath } from "./workspace-path"; + +export interface TerminalInfo { + id: string; + name: string; + workspaceRoot: string; + alive: boolean; +} + +interface TerminalSession { + id: string; + name: string; + workspaceRoot: string; + pty: IPty; + // Rolling buffer of everything the pty has emitted — capped the same way + // background-task output is, so a chatty process can't grow memory + // unboundedly over a long session. Doubles as what gets shown when a + // session is reopened after a restart (the process itself can't survive + // that, but its last output can). + scrollback: string; + alive: boolean; +} + +// Global, not per-workspace — mirrors MAX_BACKGROUND_TASKS in agent-tools.ts, +// which uses the same global-cap shape. +const MAX_TERMINALS = 5; +const MAX_SCROLLBACK_CHARS = 200_000; + +const terminals = new Map(); +let nextId = 1; + +function defaultShell(): string { + if (process.platform === "win32") return process.env.COMSPEC || "cmd.exe"; + return process.env.SHELL || "/bin/bash"; +} + +export function createTerminal( + workspaceRoot: string, + opts: { cwd?: string; name?: string } = {}, + onData: (chunk: string) => void, + onExit: (exitCode: number) => void +): { id: string; name: string } { + const runningCount = [...terminals.values()].filter((t) => t.alive).length; + if (runningCount >= MAX_TERMINALS) { + throw new Error(`Already have ${MAX_TERMINALS} terminals open — close one first.`); + } + + const cwd = resolveSafePath(workspaceRoot, opts.cwd ?? "."); + const shell = defaultShell(); + // Terminals default to allowing network — unlike a one-shot run_command + // call the model composes, this is an interactive session a human can + // type into directly and see exactly what's happening, so the tighter + // per-call default elsewhere in Agent mode would mostly just make + // ordinary shell use (curl, ping, package managers) confusingly fail. + // Filesystem confinement to the workspace still applies where the + // platform supports it. + const wrapped = wrapCommand(shell, { workspaceRoot, allowNetwork: true, cwd }); + const ptyOptions = { name: "xterm-256color", cols: 80, rows: 24, cwd, env: process.env }; + const ptyProcess = wrapped ? pty.spawn(wrapped.command, wrapped.args, ptyOptions) : pty.spawn(shell, [], ptyOptions); + + const id = `t${nextId++}`; + const session: TerminalSession = { + id, + name: opts.name?.trim() || `Terminal ${id}`, + workspaceRoot: path.resolve(workspaceRoot), + pty: ptyProcess, + scrollback: "", + alive: true, + }; + ptyProcess.onData((chunk) => { + session.scrollback += chunk; + if (session.scrollback.length > MAX_SCROLLBACK_CHARS) { + session.scrollback = session.scrollback.slice(-MAX_SCROLLBACK_CHARS); + } + onData(chunk); + }); + ptyProcess.onExit(({ exitCode }) => { + session.alive = false; + onExit(exitCode); + }); + terminals.set(id, session); + return { id, name: session.name }; +} + +function requireTerminal(id: string): TerminalSession { + const session = terminals.get(id); + if (!session) throw new Error(`No terminal with id "${id}".`); + return session; +} + +export function writeToTerminal(id: string, data: string): void { + requireTerminal(id).pty.write(data); +} + +export function resizeTerminal(id: string, cols: number, rows: number): void { + requireTerminal(id).pty.resize(Math.max(1, Math.floor(cols)), Math.max(1, Math.floor(rows))); +} + +export function readTerminalOutput(id: string, tailChars = 4000): string { + const session = requireTerminal(id); + return session.scrollback.length > tailChars ? session.scrollback.slice(-tailChars) : session.scrollback; +} + +// Removes the session entirely, unlike a plain kill — once the user (or the +// model) explicitly closes a terminal there's no reason to keep it around +// for listTerminals()/readTerminalOutput() to still surface. +export function closeTerminal(id: string): void { + const session = terminals.get(id); + if (!session) return; + if (session.alive && session.pty.pid) killProcessTree(session.pty.pid); + terminals.delete(id); +} + +// Same role as killBackgroundCommandsForWorkspace in agent-tools.ts — called +// when the renderer switches away from a workspace so terminals from the +// old one don't keep running (and don't keep counting against +// MAX_TERMINALS) until app quit. +export function closeAllForWorkspace(workspaceRoot: string): number { + const root = path.resolve(workspaceRoot); + let closed = 0; + for (const [id, session] of terminals) { + if (session.workspaceRoot !== root) continue; + closeTerminal(id); + closed++; + } + return closed; +} + +export function closeAll(): void { + for (const id of [...terminals.keys()]) closeTerminal(id); +} + +export function listTerminals(workspaceRoot?: string): TerminalInfo[] { + const root = workspaceRoot ? path.resolve(workspaceRoot) : undefined; + return [...terminals.values()] + .filter((t) => !root || t.workspaceRoot === root) + .map((t) => ({ id: t.id, name: t.name, workspaceRoot: t.workspaceRoot, alive: t.alive })); +} diff --git a/app/src/workspace-path.ts b/app/src/workspace-path.ts new file mode 100644 index 0000000..17b0a8b --- /dev/null +++ b/app/src/workspace-path.ts @@ -0,0 +1,37 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +// Every tool call (and, since phase 2, every terminal) is confined to the +// chosen workspace directory — this resolves the (possibly relative, +// possibly attacker-crafted via a prompt injection in file content the +// model read) path and throws if it would escape that directory via ../ or +// an absolute path elsewhere on disk. Lives in its own module (rather than +// inside agent-tools.ts, where it originated) so terminal-manager.ts can +// import it without creating a circular dependency between the two. +export function resolveSafePath(workspaceRoot: string, relativePath: string): string { + const root = path.resolve(workspaceRoot); + const resolved = path.resolve(root, relativePath || "."); + const isWithin = (parent: string, child: string): boolean => { + const relative = path.relative(parent, child); + return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)); + }; + if (!isWithin(root, resolved)) { + throw new Error(`Path "${relativePath}" is outside the workspace directory.`); + } + + // Lexical checks alone are bypassable through a symlink inside the + // workspace that points elsewhere. Resolve the target, or its nearest + // existing parent for new files, and verify the real path too. + const realRoot = fs.realpathSync(root); + let existing = resolved; + while (!fs.existsSync(existing)) { + const parent = path.dirname(existing); + if (parent === existing) break; + existing = parent; + } + const realExisting = fs.realpathSync(existing); + if (!isWithin(realRoot, realExisting)) { + throw new Error(`Path "${relativePath}" resolves outside the workspace directory through a symbolic link.`); + } + return resolved; +} diff --git a/app/vitest.config.ts b/app/vitest.config.ts index 022b48d..f8db4a5 100644 --- a/app/vitest.config.ts +++ b/app/vitest.config.ts @@ -10,5 +10,13 @@ export default defineConfig({ test: { environment: "node", include: ["src/**/*.test.ts"], + // Several tests spawn a real child process or PTY and poll for + // observable output (agent-tools.test.ts, resource-monitor.test.ts, + // terminal-manager.test.ts). Vitest's 5000ms default has repeatedly + // been too tight for that on Windows CI runners, where process/PTY + // spawn is noticeably slower than on Linux/macOS — this raises the + // floor for the whole suite instead of chasing each Windows-only + // flake one file at a time as it turns up. + testTimeout: 20_000, }, }); diff --git a/docs/AGENT_MODE.md b/docs/AGENT_MODE.md new file mode 100644 index 0000000..bda9dde --- /dev/null +++ b/docs/AGENT_MODE.md @@ -0,0 +1,168 @@ +# Agent mode + +Agent mode lets the model read/write files and run commands in a folder you explicitly choose +("the workspace"), with every action gated behind your approval. This document is the detailed +reference; see the [README](../README.md#agent-mode) for a shorter overview. + +Source of truth for all of this is `app/src/agent-tools.ts` (tool implementations + the +destructive-command blocklist), `app/src/command-sandbox.ts` (OS-level sandboxing), and +`frontend/src/lib/tool-approval.ts` (which tools may be auto-approved). + +## Starting a session + +Click **Agent** in the chat toolbar and pick a folder. That folder becomes `workspaceRoot` for the +rest of the conversation — every filesystem tool call is resolved and validated against it before +anything touches disk. + +## Tool catalog + +Every tool below is dispatched through a single `executeTool(workspaceRoot, name, args)` switch in +`agent-tools.ts`, so this table reflects the actual current tool set rather than a curated subset. + +**Filesystem (workspace-sandboxed)** + +| Tool | What it does | +|---|---| +| `read_file` | Read a text file, optionally by line range | +| `write_file` | Create or overwrite a file (creates parent directories as needed); shows a line-by-line diff before approval | +| `replace_in_file` | Replace an exact text block without rewriting the whole file, optionally all occurrences | +| `find_files` | Discover files by glob pattern | +| `file_info` | Inspect path metadata (size, type, mtime) | +| `list_dir` | List files and subdirectories | +| `search_files` | Search for a text string across the workspace | +| `find_symbol_references` | Search for references to a symbol name (a scoped variant of `search_files`) | +| `make_directory` | Create a directory | +| `move_path` | Move/rename a file or directory | +| `delete_path` | Delete a file or directory (optionally recursive) | +| `apply_patch` | Apply a unified-diff-style patch across one or more files | +| `read_notes` / `write_notes` | Read/write a per-workspace scratch note file the model can use as working memory across a long task | + +**Execution** + +| Tool | What it does | +|---|---| +| `run_command` | Execute a shell command in the workspace (or a subfolder), with a 60s timeout and network access off by default | +| `run_code` | Run a Python or JavaScript snippet — a convenience over shell-quoting multi-line code through `run_command`, not a new capability | +| `start_background_command` / `get_background_output` / `stop_background_command` / `list_background_commands` | Launch a long-running command (e.g. a dev server) without blocking the agent loop, and poll or stop it later | +| `create_terminal` / `write_to_terminal` / `read_terminal_output` / `close_terminal` | Drive a PTY-backed terminal the model can interact with by polling output rather than streaming; the human-facing terminal panel uses a separate IPC path and is unaffected | + +**Git** + +| Tool | What it does | +|---|---| +| `git_status`, `git_diff`, `git_log` | Read-only git helpers (auto-approvable) so the model doesn't need to guess flag syntax | +| `git_commit` | Stage everything and commit — requires approval, like `write_file` | + +**Network** (see [Network tools and the network toggle](#network-tools-and-the-network-toggle)) + +| Tool | What it does | +|---|---| +| `fetch_url` | Fetch a URL and return its content | +| `http_request` | A general HTTP request with method/headers/body, for calling APIs | +| `web_search` | Run a web search | +| `capture_page_screenshot` | Render a URL and save a screenshot into the workspace | +| `github_list_repositories` | List repositories accessible to the linked GitHub account | +| `github_repository_tree` | Inspect a repository's complete file structure | +| `github_read_file` | Read a file from a public or private linked-account repository | + +## Safety model + +**Workspace confinement.** The filesystem tools resolve every path against `workspaceRoot` and +reject anything that would escape it — `../../etc`, an absolute path elsewhere on disk, or a +symlink that resolves outside the workspace. This confinement is real for those tools specifically. + +**`run_command` and `run_code` are different.** A shell command (or a script handed to +`python3`/`node`) is opaque text that can reference any path on the system regardless of its +working directory, so neither is confined the way the file tools are. Two layers of mitigation +apply instead: + +1. **OS-level sandboxing where available** (`command-sandbox.ts`): on Linux, commands run inside + [bubblewrap](https://github.com/containers/bubblewrap) (`bwrap`) when it's installed *and* + verified to actually work — a plain `which bwrap` isn't trusted, since modern Ubuntu (23.10+) + restricts unprivileged user namespaces by AppArmor policy and would make `bwrap` fail silently + at runtime. On macOS, `sandbox-exec` is used (built in, no install needed). **Windows has no + equivalent lightweight primitive** — there's no OS-level confinement there, only the blocklist + below plus `resource-monitor.ts` limits. Query current capabilities via `agent:getSandboxCapabilities`. +2. **A destructive-command blocklist**, checked regardless of sandboxing and even against + already-approved commands: deleting outside the workspace, formatting a drive, shutting down the + machine, registry deletion, `sudo`/`runas`, or piping a remote script into a shell are **rejected + outright**. This catches the common catastrophic cases, not everything a shell or script can do — + only approve a command or snippet you actually understand. + +**Explicit approval, every time (with one narrow exception).** Every tool call shows an +Allow/Deny card before it executes. `frontend/src/lib/tool-approval.ts` allows a narrow set of +*strictly read-only, no-network* tools to be marked "always allow this session": + +``` +read_file, find_files, file_info, list_dir, search_files, +git_status, git_diff, git_log, read_notes, +get_background_output, list_background_commands, read_terminal_output +``` + +The bar for that list is not "doesn't write to the workspace" — it's "the user has nothing to lose +by never being asked again." That's deliberately narrower than "read-only": `web_search`, +`fetch_url`, `http_request`, and the `github_*` tools all send a request to a destination the model +chooses, which is an unattended outbound channel for whatever the model has already read if +standing-approved — so they always prompt. `capture_page_screenshot` fetches a model-chosen URL +*and* writes into the workspace, so it's excluded for the same reason. Filesystem mutations, +`run_command`, `run_code`, and `git_commit` always require a fresh click, since they have real, +potentially irreversible effects. The trust list is in-memory only — closing and reopening a chat +resets it. + +This is a default-deny design: a tool added to the catalog without being explicitly added to +`AUTO_APPROVABLE_TOOLS` requires per-call approval until someone deliberately decides otherwise. + +**Untrusted model input.** Agent mode reads file contents and web pages, and instructions embedded +in that content can attempt to steer the model into calling tools it wasn't asked to. The per-call +approval prompt is the actual control against this becoming unattended action — it's why the +auto-approve list above is scoped so conservatively. + +**Step limit.** A per-turn cap of 25 tool-result → model-continuation round trips stops a model +from looping indefinitely without producing a final answer. + +## Network tools and the network toggle + +Settings has a global switch to turn off network-capable tools entirely +(`networkToolsEnabled: false`). When off, any tool in the network set above throws immediately +instead of running, regardless of approval state — useful if you want Agent mode confined to a +fully offline, filesystem-and-shell-only workflow. + +## Preview & rollback + +- A pending `write_file` call shows a real line-by-line diff against the file's current content + (or a "new file" badge if it doesn't exist yet), so you see exactly what would change before + clicking Allow. +- **Undo last edit** reverts the most recent applied `write_file`, restoring the previous content + or deleting the file if the edit created it. History is per-workspace, capped at the last 20 + writes, and lives only in memory for the running session — not a durable version history. + +## Quick actions + +If the workspace has `test`/`lint`/`format` scripts in its `package.json`, **Run Tests**, **Lint**, +and **Format** buttons appear in the toolbar. They run the corresponding `npm` script directly +(reusing the same sandboxing as `run_command`) and drop the output into the chat without going +through the model at all — useful for a fast check that doesn't cost a model turn. + +## MCP (Model Context Protocol) servers + +Add external MCP servers in Settings to give Agent mode extra tools beyond the built-in catalog — +anything from a database query tool to a browser-automation server. Two transports are supported: + +- **stdio** — launches a local command (e.g. `npx -y @modelcontextprotocol/server-filesystem + /some/path`) and speaks JSON-RPC over its stdin/stdout. +- **HTTP** — connects to a remote MCP server's "Streamable HTTP" endpoint. + +Enabled servers reconnect automatically on launch. Each server's tools appear in Agent mode's tool +list prefixed with the server's name, and go through the exact same Allow/Deny approval flow as +built-in tools — an MCP server is not a trust escalation. SSE and plain WebSocket transports +aren't implemented: SSE is the legacy MCP HTTP transport, now superseded by Streamable HTTP, and +WebSocket was never part of the MCP spec itself. + +## Model choice matters + +Agent mode works with whatever model you point it at, but only actually produces tool calls if +that model was trained for function/tool calling — a model without that training will just chat +normally and never call a tool. The Settings model browser flags models with reliable tool-calling +support with a 🔧 **Tool calling** badge (e.g. the Qwen3 family, Llama 3.1+, Mistral Nemo, +Qwen2.5-Coder, Devstral). The llama.cpp backend does not yet wire up tool-calling for Agent mode — +a clear error explains this if you try. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..8b886fb --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,170 @@ +# Architecture + +Modelforge is an Electron desktop app split into three independently-buildable packages, plus a +standalone Python project used offline to train a small model: + +``` +frontend/ React 19 + Vite renderer — everything the user sees +app/ Electron main process — window/IPC/OS integration, provider adapters, persistence +lib/ Rust native addon (napi-rs) — the GGUF downloader's parallel-range HTTP engine +ml/ Standalone Python project — trains the hardware/model-fit recommender offline +``` + +`frontend` and `app` are separate npm workspaces (separate `package.json`, separate +`node_modules`) glued together only by the IPC bridge described below and by the build step that +copies `frontend/dist` into the packaged app as a static asset. `lib` is a separate Cargo crate +compiled to a native Node addon and copied into `app/native`. `ml`'s *training pipeline* is not +built by `npm run build:all` or run in CI — it's a separate, manual, offline step — but its +*output* (a small trained `.pt` file) is bundled as a resource (see `extraResources` / +`python` in `app/package.json`) and consumed at runtime by a managed Python worker; see +[ml/hardware-recommender/](#mlhardware-recommender) below for exactly how. + +## Process model + +Electron's two-process split is used exactly as intended, with nothing loosened for convenience: + +- **Main process** (`app/src/main.ts`) — a normal Node.js process with full OS access. Owns the + `BrowserWindow`, registers every `ipcMain.handle` channel, and is the only place that talks to + Ollama, cloud provider APIs, the filesystem, child processes, and the OS keychain. +- **Renderer process** (`frontend/`) — the UI, running with `contextIsolation: true` and + `nodeIntegration: false`. It has no direct access to Node or Electron APIs. +- **Preload script** (`app/src/preload.ts`) — the only bridge between them. It uses + `contextBridge.exposeInMainWorld` to expose a fixed, typed `window.electronAPI` surface + (see `frontend/src/types/electron.d.ts` for the renderer-side type). The renderer can only ever + call the specific functions this file exposes — there is no generic "run arbitrary IPC channel" + escape hatch. + +This means every capability the UI has — reading a file, starting Ollama, calling a cloud +provider, writing a setting — exists because a corresponding `ipcMain.handle("namespace:action", …)` +was deliberately added in `main.ts` and mirrored in `preload.ts`. Grep for `ipcMain.handle` in +`app/src/main.ts` for the authoritative, current list of every capability the renderer has; the +channel names are namespaced by feature (`ollama:*`, `sessions:*`, `agent:*`, `downloads:*`, +`data:*`, and so on). + +Streaming responses (chat tokens, download progress, terminal output) use the complementary +`event.sender.send(...)` / `ipcRenderer.on(...)` push channel rather than `invoke`, since `invoke` +is request/response only. + +## Renderer (`frontend/`) + +- **Routing / pages** (`src/pages/`): `Chat.tsx` (the main conversation view and Agent mode UI), + `Settings.tsx` (by far the largest file in the app — every settings tab lives here), + `Compare.tsx` (side-by-side multi-model prompting), `DownloadCenter.tsx`, `RuntimeManager.tsx`, + `UsageDashboard.tsx`. +- **Shared components** (`src/components/`): layout chrome, the command palette, Markdown/Mermaid + rendering, the onboarding wizard, the embedded terminal panel, and `ui/` — generated + [shadcn](https://ui.shadcn.com/)-style primitives on top of Base UI. +- **`src/lib/`** — logic that isn't a component: i18n (`i18n.tsx`, `translations.ts`), the model + catalog and pricing tables (`model-catalog.ts`, `pricing.ts`), provider capability metadata + (`providers.ts`), context-window compaction (`context-compaction.ts`), keybindings, tool-approval + policy (`tool-approval.ts` — see [Agent mode](AGENT_MODE.md)), and `sessions-context.tsx`, the + React context that owns in-memory chat/session state and talks to the main process through + `window.electronAPI`. + +The renderer never talks to Ollama, OpenAI, Anthropic, etc. directly — every provider call is +proxied through the main process so API keys never need to reach the renderer's JS context, and so +CSP can block the renderer from making arbitrary network requests at all. + +Production builds inline everything (JS, CSS, even encoded assets) into a single `index.html` via +`vite-plugin-singlefile`, so the packaged app can load the UI directly from `file://` without a +local HTTP server — matching how it's actually loaded in production, so `npm run dev` and a +packaged build exercise the same loading path in spirit. + +## Main process (`app/src/`) + +Roughly grouped by responsibility: + +| Area | Files | +|---|---| +| Bootstrap | `main.ts` (window creation, all IPC registration), `menu.ts`, `preload.ts` | +| Provider adapters | `providers/openai.ts`, `providers/anthropic.ts`, `providers/gemini.ts`, `providers/openai-compatible.ts` (shared implementation for every OpenAI-compatible custom endpoint), `providers/sse.ts` (shared streaming parser), `providers/types.ts` | +| Local model runtimes | `ollama-manager.ts` (start/stop/pull/list against a local or remote Ollama daemon), `llamacpp-manager.ts` (GGUF inference via node-llama-cpp, GPU backend detection), `local-server-manager.ts` (custom OpenAI-compatible local backends: vLLM, LocalAI, TGI, etc.) | +| Agent mode | `agent-tools.ts` (every tool implementation, workspace sandboxing, the destructive-command blocklist), `command-sandbox.ts` (OS-level sandboxing via bubblewrap/`sandbox-exec` where available), `terminal-manager.ts` (the embedded PTY-backed terminal panel), `mcp-client.ts` (external MCP server connections) | +| Persistence | `json-store.ts` (shared atomic-write/corruption-recovery helper), `settings-store.ts`, `sessions-store.ts`, `projects-store.ts`, `secrets-store.ts` (OS-keychain-backed via Electron `safeStorage`), `scheduled-tasks-store.ts`, `download-jobs-store.ts`, `energy-usage-store.ts` | +| Retrieval | `rag.ts` / `rag-db.ts` — chunking, Ollama-embedding, and similarity search for large folder/file attachments | +| Models & downloads | `huggingface.ts` (Hub search API), `download-queue.ts` / `download-worker.ts` / `native-downloader.ts` (resumable parallel downloads, using the Rust addon when available), `download-verification.ts` (checksum/size verification) | +| System | `system-specs.ts` (RAM/VRAM detection across all GPUs), `resource-monitor.ts`, `power-monitor.ts`, `process-tree.ts`, `benchmark-runner.ts` | +| Integrations | `figma.ts`, `ocr.ts` (tesseract.js), `accounts.ts` (linked GitHub account), `scheduler.ts` (scheduled chat-completion tasks) | +| Misc | `logger.ts` (rotating file logs surfaced in Settings → Diagnostics), `updater.ts` (GitHub Releases auto-update), `data-transfer.ts` (export/import), `keybindings.ts`, `python-runtime-manager.ts` (manages the bundled Python workers under `app/python/`) | + +### Persistence pattern + +All app state (settings, sessions, projects, scheduled tasks, download jobs, energy history) is +plain JSON files under Electron's `userData` directory, going through the shared helpers in +`json-store.ts`: + +- **Atomic writes** — write to a temp file, then rename over the real one. A rename is atomic on + both Windows and POSIX filesystems, so a crash or power loss mid-write can never leave a reader + looking at a half-written file. +- **Corruption recovery** — if an existing file fails to parse as JSON, it's copied aside as + `.corrupted-` and logged, rather than silently discarded. This means the next + write can't accidentally overwrite salvageable data with a blank slate. + +Secrets (provider API keys) go through `secrets-store.ts` instead, which encrypts values at rest +via Electron's `safeStorage` (backed by the OS credential store — Keychain, DPAPI, or +libsecret/kwallet) rather than writing them out as plain JSON. On a system with no OS credential +store available (`safeStorage.isEncryptionAvailable()` returns `false` — some keyring-less Linux +setups), the store falls back to writing the value unencrypted rather than silently dropping the +key; this fallback is logged at warn level, and `secrets:isEncryptionAvailable` lets the renderer +check the same condition to show a plaintext-storage warning in Settings instead of the normal +"encrypted at rest" note. + +## Native addon (`lib/`) + +A small Rust crate (`modelforge-native`), compiled with [napi-rs](https://napi.rs/) into a +platform-specific `.node` addon that `app/src` loads like any other Node module (see +`app/native/index.js`). Its only current job is `download_gguf_file`: resuming an interrupted +Hugging Face GGUF download and using parallel HTTP Range-request connections when the server and +file size support it, falling back to a single stream otherwise. It mirrors the signature and +observable behavior of the TypeScript downloader it can replace, so `download-queue.ts` can use +either implementation interchangeably depending on availability. + +Build it with `npm run build:native --prefix app` (invokes `napi build` in `lib/`, which requires +a Rust toolchain); `npm run build:all --prefix app` does this as part of a full production build. +The prebuilt `.node` binaries checked into `app/native/` mean a plain `npm run dev` does **not** +require Rust installed — only rebuilding the native layer itself does. + +## `ml/hardware-recommender/` + +A standalone Python *training* project — see its own [README](../ml/hardware-recommender/README.md) +for details — that trains a small PyTorch multi-task model predicting hardware/model fit from the +public Open LLM Leaderboard catalog plus transparent memory/throughput heuristics (there's no +measured-compatibility dataset publicly available). Its trained output, `artifacts/hardware_recommender.pt`, *is* consumed at runtime — but through a +committed copy, not a live link to this directory: the checkpoint is duplicated to +`app/python/artifacts/hardware_recommender.pt`, and `app/python/recommender_worker.py` is a +trimmed, inference-only, self-contained reimplementation of this project's model architecture and +feature encoding (deliberately *not* importing `ml/hardware-recommender/recommend.py`, so the +packaged `app/python/` directory doesn't need this project's pandas/pyarrow training dependencies +just to run inference). `app/src/system-specs.ts` spawns that worker as a long-lived JSON-line +process via `app/src/python-runtime-manager.ts`'s `ManagedPythonWorker`, and calls +`recommendModelsWithML(...)` to enhance the plain-heuristic recommendation for each catalog model +with the trained model's prediction. If the worker isn't available (Python not installed, or the +worker is briefly unhealthy), `recommendModelsWithML` falls back to the pure heuristic per model +rather than blocking the recommendation — the ML path is a strict enhancement, never a dependency. +Retraining here and re-copying the updated `.pt`/worker logic into `app/python/` is a manual, +offline step — it is **not** part of `npm run build:all` or CI, so the two copies can drift if a +model change here isn't manually ported over. + +## Data flow: sending a chat message + +1. Renderer (`Chat.tsx` via `sessions-context.tsx`) calls `window.electronAPI.chat.send(...)`. +2. Preload forwards this to the `chat:send` IPC channel. +3. `main.ts` resolves the active provider (`ollama` / `llamacpp` / `openai` / `anthropic` / + `gemini` / a custom OpenAI-compatible endpoint), decrypts its API key from `secrets-store` if + needed, and calls the matching adapter in `providers/`. +4. The adapter opens a streaming HTTP request to the provider and, for each chunk, calls back into + `main.ts`, which forwards it to the renderer over a per-request push channel + (`chat:chunk:${requestId}`). +5. `sessions-context.tsx` appends chunks to the in-progress assistant message as they arrive and + persists the finished message to `sessions-store.ts` once the stream ends. +6. If Agent mode is active and the model requests a tool call, the same request/response cycle + happens per tool call via `agent:executeTool` (gated behind the Allow/Deny UI — see + [Agent mode](AGENT_MODE.md)) before the conversation continues. + +Cancellation works the same way in reverse: `chat:cancel` looks up the request's `AbortController` +in `main.ts`'s `activeChatRequests` map and aborts the in-flight HTTP request. + +## See also + +- [Agent mode](AGENT_MODE.md) — full tool list, sandboxing model, and approval flow +- [Development](DEVELOPMENT.md) — setup, running, testing, building, and releasing diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..aa07cf7 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,146 @@ +# Development guide + +## Prerequisites + +- [Node.js](https://nodejs.org) 22+ +- A Rust toolchain (`cargo`) — only needed if you plan to rebuild the native downloader addon in + `lib/`. Prebuilt binaries are checked into `app/native/`, so a normal `npm run dev` does not + require Rust. +- [Ollama](https://ollama.com) installed locally if you want to exercise the default provider + end-to-end during development (not required to build or test the app). + +## Getting the code running + +```sh +git clone https://github.com/voidstackloop/modelforge.git +cd modelforge + +npm install --prefix frontend +npm install --prefix app +``` + +`frontend` and `app` are independent npm projects — always pass `--prefix` (or `cd` into the +directory) rather than expecting a single top-level `npm install` to cover both. + +### Development mode + +```sh +npm run dev --prefix app +``` + +This runs three processes concurrently (`app/package.json`'s `dev` script, via `concurrently`): + +1. `dev:renderer` — `vite` dev server for `frontend/`, on port 5173 with hot module reload. +2. `dev:main` — `tsc --watch` compiling `app/src/*.ts` to `app/dist/`. +3. `dev:electron` — waits for both of the above (`wait-on tcp:5173 dist/main.js`), then launches + Electron pointed at the Vite dev server instead of the built `frontend/dist`. + +Editing renderer code hot-reloads via Vite. Editing main-process code (`app/src/*.ts`) recompiles +automatically but requires restarting the Electron process to take effect (`tsc --watch` doesn't +restart Electron for you). + +## Testing + +```sh +npm test --prefix frontend +npm test --prefix app +``` + +Both use [Vitest](https://vitest.dev/). Test files sit next to the code they cover +(`foo.ts` → `foo.test.ts`), not in a separate `__tests__` tree. + +- **Frontend tests** cover pure logic in `src/lib/` — pricing math, context compaction, keybinding + parsing, provider capability lookups, tool-approval policy — rather than component rendering. +- **App tests** cover the store layer (atomic writes, corrupted-file recovery — see + [Architecture: persistence pattern](ARCHITECTURE.md#persistence-pattern)), the agent tools + (including path-traversal rejection and shell command execution), sandbox capability detection, + and the RAG chunking/similarity logic. +- The Rust addon has its own test suite: `cd lib && cargo test` (mocks HTTP with `wiremock`). + +Both `npm test` suites, plus lint, typecheck, and build for both packages, run in CI on every push +and pull request via [`.github/workflows/ci.yml`](../.github/workflows/ci.yml). Run the same +checks locally before opening a PR: + +```sh +npm run lint --prefix frontend +npm run build --prefix frontend +npm run build --prefix app +npm test --prefix frontend +npm test --prefix app +``` + +## Building a full production bundle + +```sh +npm run build:all --prefix app +``` + +Runs, in order: `lib` native build (`napi build --platform --release`, output to `app/native/`) → +`frontend` production build (`tsc -b && vite build`, output to `frontend/dist/`) → `app` TypeScript +build (`tsc -p tsconfig.json`, output to `app/dist/`). This is what CI and `npm run package` both +build on top of. + +## Packaging an installer + +```sh +npm run package --prefix app +``` + +Runs `build:all` and then [electron-builder](https://www.electron.build/), producing a +platform-specific installer in `app/release/`: + +| Platform | Output | electron-builder target | +|---|---|---| +| Windows | `Modelforge Setup *.exe` | NSIS (default) | +| macOS | `Modelforge-*.dmg` / `Modelforge-*-arm64.dmg` | `dmg`, both Intel and Apple Silicon | +| Linux | `Modelforge-*.AppImage` | `AppImage` | + +Cross-compiling installers for another OS generally doesn't work with electron-builder — build on +(or in CI, via) the target platform. [`.github/workflows/release.yml`](../.github/workflows/release.yml) +builds all three in parallel (matrix over `windows-latest` / `macos-latest` / `ubuntu-latest`) when +a `v*.*.*` tag is pushed, and attaches the resulting installers to a GitHub Release — no signing +certificate is configured yet, so every platform's installer shows an "unknown publisher" warning +on first run; see the [README](../README.md#installation) for what that looks like for a user. + +## Rebuilding the native addon + +Only needed if you change `lib/src/*.rs`: + +```sh +npm run build:native --prefix app +``` + +Equivalent to `cd lib && npm install && npm run build`, which invokes +`napi build --platform --release --output-dir ../app/native`. `napi`'s target list +(`lib/package.json`) covers `x86_64-pc-windows-msvc`, `x86_64-apple-darwin`, +`aarch64-apple-darwin`, and `x86_64-unknown-linux-gnu` — building for a target other than your +current host requires the matching Rust cross-compilation target installed. + +## Project structure + +``` +frontend/ React + Vite renderer (the UI) + src/pages/ Chat and Settings screens, Compare, Download Center, Runtime Manager, Usage Dashboard + src/components/ Shared UI (layout, command palette, markdown/mermaid rendering, terminal panel, shadcn primitives) + src/lib/ i18n, model catalogs, pricing estimates, provider helpers, tool-approval policy + +app/ Electron main process + src/main.ts Window management, IPC handler registration (grep "ipcMain.handle" for the full API surface) + src/preload.ts The typed contextBridge surface exposed to the renderer as window.electronAPI + src/providers/ Ollama/OpenAI/Anthropic/Gemini/OpenAI-compatible chat + tool-calling adapters + src/agent-tools.ts Agent mode's full tool catalog (workspace-sandboxed filesystem/shell/git/network tools) + src/command-sandbox.ts OS-level command sandboxing (bubblewrap on Linux, sandbox-exec on macOS) + src/*-store.ts Settings/sessions/projects/secrets/scheduled-tasks persistence (atomic writes, corruption recovery) + src/rag.ts Chunking + embedding + retrieval for large folder attachments + src/logger.ts Rotating file logs surfaced via Settings → Data → Diagnostics + +lib/ Rust native addon (napi-rs) — resumable, parallel-range GGUF downloads + src/lib.rs napi bindings + src/manager.rs Download orchestration + src/download/ HTTP range-request logic + +ml/hardware-recommender/ Standalone offline Python project — see its own README +``` + +See [Architecture](ARCHITECTURE.md) for how these pieces fit together and communicate, and +[Agent mode](AGENT_MODE.md) for the tool catalog and sandboxing model in detail. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 868a7fe..91ef82b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,6 +11,8 @@ "@base-ui/react": "^1.6.0", "@fontsource-variable/geist": "^5.2.9", "@tailwindcss/vite": "^4.3.2", + "@xterm/addon-fit": "^0.11.0", + "@xterm/xterm": "^6.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -3486,6 +3488,21 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@xterm/addon-fit": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.11.0.tgz", + "integrity": "sha512-jYcgT6xtVYhnhgxh3QgYDnnNMYTcf8ElbxxFzX0IZo+vabQqSPAjC3c1wJrKB5E19VwQei89QCiZZP86DCPF7g==", + "license": "MIT" + }, + "node_modules/@xterm/xterm": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-6.0.0.tgz", + "integrity": "sha512-TQwDdQGtwwDt+2cgKDLn0IRaSxYu1tSUjgKarSDkUM0ZNiSRXFpjxEsvc/Zgc5kq5omJ+V0a8/kIM2WD3sMOYg==", + "license": "MIT", + "workspaces": [ + "addons/*" + ] + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 71edf6d..6fb8f4a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,8 @@ "@base-ui/react": "^1.6.0", "@fontsource-variable/geist": "^5.2.9", "@tailwindcss/vite": "^4.3.2", + "@xterm/addon-fit": "^0.11.0", + "@xterm/xterm": "^6.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e913d82..4403ce6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,6 +4,8 @@ import Chat from "./pages/Chat"; import Settings from "./pages/Settings"; import Compare from "./pages/Compare"; import UsageDashboard from "./pages/UsageDashboard"; +import DownloadCenter from "./pages/DownloadCenter"; +import RuntimeManager from "./pages/RuntimeManager"; import { ThemeProvider } from "@/components/theme-provider"; import { ToastProvider } from "@/components/toast"; import { SessionsProvider } from "@/lib/sessions-context"; @@ -19,6 +21,8 @@ const router = createHashRouter([ { path: "settings", element: }, { path: "compare", element: }, { path: "usage", element: }, + { path: "downloads", element: }, + { path: "runtimes", element: }, ], }, ]); diff --git a/frontend/src/components/layout.tsx b/frontend/src/components/layout.tsx index a4ef282..1ca6f6f 100644 --- a/frontend/src/components/layout.tsx +++ b/frontend/src/components/layout.tsx @@ -20,6 +20,8 @@ import { BarChart3, Menu, Sparkles, + Download, + Server, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -740,6 +742,19 @@ export default function Layout() { {t.usageDashboard} + +
diff --git a/frontend/src/components/terminal-panel.tsx b/frontend/src/components/terminal-panel.tsx new file mode 100644 index 0000000..76607f7 --- /dev/null +++ b/frontend/src/components/terminal-panel.tsx @@ -0,0 +1,167 @@ +import { useEffect, useRef, useState } from "react"; +import { Terminal as XTerm } from "@xterm/xterm"; +import { FitAddon } from "@xterm/addon-fit"; +import "@xterm/xterm/css/xterm.css"; +import { Plus, X, TerminalSquare } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; + +interface OpenTerminal { + id: string; + name: string; + alive: boolean; +} + +// A human-facing interactive terminal panel — separate from the model's +// create_terminal/write_to_terminal/etc. tool calls (which poll instead of +// streaming; see terminal-manager.ts). Keeps exactly one xterm.js instance +// mounted and swaps which terminal's buffered + live output feeds into it +// when switching tabs, rather than keeping N instances mounted at once. +export function TerminalPanel({ workspaceRoot, onClose }: { workspaceRoot: string; onClose: () => void }) { + const [terminals, setTerminals] = useState([]); + const [activeId, setActiveId] = useState(null); + const containerRef = useRef(null); + const xtermRef = useRef(null); + const fitRef = useRef(null); + const activeIdRef = useRef(null); + // Full output ever seen per terminal, so switching tabs can replay it — + // xterm itself only holds whatever's currently written to its one + // mounted instance. + const buffersRef = useRef>(new Map()); + + useEffect(() => { + if (!containerRef.current) return; + const term = new XTerm({ + convertEol: true, + fontSize: 13, + fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace", + theme: { background: "#00000000" }, + }); + const fit = new FitAddon(); + term.loadAddon(fit); + term.open(containerRef.current); + fit.fit(); + xtermRef.current = term; + fitRef.current = fit; + + term.onData((data) => { + if (activeIdRef.current) window.api.terminal.write(activeIdRef.current, data); + }); + + const resizeObserver = new ResizeObserver(() => { + fit.fit(); + if (activeIdRef.current) window.api.terminal.resize(activeIdRef.current, term.cols, term.rows); + }); + resizeObserver.observe(containerRef.current); + + return () => { + resizeObserver.disconnect(); + term.dispose(); + xtermRef.current = null; + }; + // Intentionally mount once — the panel keeps one xterm instance for + // its whole lifetime and re-targets it on tab switches instead. + }, []); + + function handleData(id: string, chunk: string) { + buffersRef.current.set(id, (buffersRef.current.get(id) ?? "") + chunk); + if (activeIdRef.current === id) xtermRef.current?.write(chunk); + } + + function handleExit(id: string, exitCode: number) { + setTerminals((prev) => prev.map((t) => (t.id === id ? { ...t, alive: false } : t))); + const note = `\r\n\x1b[90m[process exited with code ${exitCode}]\x1b[0m\r\n`; + buffersRef.current.set(id, (buffersRef.current.get(id) ?? "") + note); + if (activeIdRef.current === id) xtermRef.current?.write(note); + } + + function switchTo(id: string) { + setActiveId(id); + activeIdRef.current = id; + xtermRef.current?.reset(); + const buffered = buffersRef.current.get(id); + if (buffered) xtermRef.current?.write(buffered); + if (fitRef.current) fitRef.current.fit(); + } + + async function createNewTerminal() { + let id = ""; + const created = await window.api.terminal.create( + workspaceRoot, + {}, + (chunk) => handleData(id, chunk), + (exitCode) => handleExit(id, exitCode) + ); + id = created.id; + buffersRef.current.set(id, ""); + setTerminals((prev) => [...prev, { id, name: created.name, alive: true }]); + switchTo(id); + } + + async function closeTerminalTab(id: string) { + await window.api.terminal.close(id); + buffersRef.current.delete(id); + setTerminals((prev) => { + const remaining = prev.filter((t) => t.id !== id); + if (activeIdRef.current === id) { + const next = remaining.at(-1); + if (next) switchTo(next.id); + else { + setActiveId(null); + activeIdRef.current = null; + xtermRef.current?.reset(); + } + } + return remaining; + }); + } + + // Opens with one terminal already running rather than an empty panel. + // Intentional one-shot on mount — createNewTerminal's own setState call + // happens asynchronously after the IPC round-trip, not synchronously + // within this effect, and it's deliberately not in the deps array since + // this should run exactly once regardless of how `terminals` changes + // afterward. + /* eslint-disable react-hooks/set-state-in-effect, react-hooks/exhaustive-deps */ + useEffect(() => { + if (terminals.length === 0) void createNewTerminal(); + }, []); + /* eslint-enable react-hooks/set-state-in-effect, react-hooks/exhaustive-deps */ + + return ( +
+
+ +
+ {terminals.map((t) => ( + + ))} +
+ + +
+
+
+ ); +} diff --git a/frontend/src/lib/model-catalog.ts b/frontend/src/lib/model-catalog.ts index d38d5fc..bba0e62 100644 --- a/frontend/src/lib/model-catalog.ts +++ b/frontend/src/lib/model-catalog.ts @@ -28,3 +28,11 @@ export const EXTRA_MODELS: CatalogEntry[] = [ { name: "qwen2.5:1.5b", label: "Qwen 2.5 1.5B", description: "Small, fast general-purpose model." }, { name: "gemma2:27b", label: "Gemma 2 27B", description: "Larger Gemma 2 for higher-quality responses." }, ]; + +// Ollama embedding models offered in the RAG "Model runtime" settings picker — +// separate from EXTRA_MODELS since these aren't chat models at all. +export const EMBEDDING_MODELS: CatalogEntry[] = [ + { name: "nomic-embed-text", label: "Nomic Embed Text", description: "Text embedding model, not for chat — used for search/RAG." }, + { name: "mxbai-embed-large", label: "MxBai Embed Large", description: "Larger, higher-quality embedding model — slower to index and query." }, + { name: "all-minilm", label: "All-MiniLM", description: "Small, fast embedding model — good for low-resource machines." }, +]; diff --git a/frontend/src/lib/providers.test.ts b/frontend/src/lib/providers.test.ts index a78a6ef..290e43c 100644 --- a/frontend/src/lib/providers.test.ts +++ b/frontend/src/lib/providers.test.ts @@ -25,6 +25,15 @@ describe("formatModelRef / parseModelRef", () => { expect(parseModelRef(formatModelRef("gemini", "gemini-2.5-pro"))?.provider).toBe("gemini"); expect(parseModelRef(formatModelRef("custom", "abc::model-x"))?.provider).toBe("custom"); }); + + it("round-trips every managed local runtime", () => { + for (const provider of ["mlx", "rocm", "vllm"] as const) { + expect(parseModelRef(formatModelRef(provider, "publisher/model"))).toEqual({ + provider, + modelId: "publisher/model", + }); + } + }); }); describe("formatCustomModelRef / parseCustomModelId", () => { diff --git a/frontend/src/lib/providers.ts b/frontend/src/lib/providers.ts index 7a66328..af6835f 100644 --- a/frontend/src/lib/providers.ts +++ b/frontend/src/lib/providers.ts @@ -82,7 +82,7 @@ export function parseCustomModelId(modelId: string): { customProviderId: string; return { customProviderId: modelId.slice(0, sep), actualModel: modelId.slice(sep + 2) }; } -const VALID_PROVIDERS: ProviderId[] = ["ollama", "openai", "anthropic", "llamacpp", "gemini", "custom", "mlx", "rocm"]; +const VALID_PROVIDERS: ProviderId[] = ["ollama", "openai", "anthropic", "llamacpp", "gemini", "custom", "mlx", "rocm", "vllm"]; export function parseModelRef(ref: string): { provider: ProviderId; modelId: string } | null { const sepIndex = ref.indexOf(":"); diff --git a/frontend/src/lib/tool-approval.test.ts b/frontend/src/lib/tool-approval.test.ts new file mode 100644 index 0000000..272eeeb --- /dev/null +++ b/frontend/src/lib/tool-approval.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect } from "vitest"; +import { canAlwaysAllow } from "./tool-approval"; + +describe("canAlwaysAllow", () => { + it("allows a standing grant for workspace-local reads", () => { + expect(canAlwaysAllow("read_file")).toBe(true); + expect(canAlwaysAllow("list_dir")).toBe(true); + expect(canAlwaysAllow("git_diff")).toBe(true); + expect(canAlwaysAllow("find_symbol_references")).toBe(true); + }); + + // A standing grant on these would let content the model reads steer an + // outbound request on every later turn with no prompt shown. + it("never allows a standing grant for tools that reach the network", () => { + expect(canAlwaysAllow("fetch_url")).toBe(false); + expect(canAlwaysAllow("web_search")).toBe(false); + expect(canAlwaysAllow("github_read_file")).toBe(false); + expect(canAlwaysAllow("github_list_repositories")).toBe(false); + expect(canAlwaysAllow("github_repository_tree")).toBe(false); + }); + + // Not read-only in either sense: it fetches a model-chosen URL and writes + // a PNG into the workspace. + it("never allows a standing grant for capture_page_screenshot", () => { + expect(canAlwaysAllow("capture_page_screenshot")).toBe(false); + }); + + it("never allows a standing grant for tools with side effects", () => { + for (const tool of ["write_file", "run_command", "run_code", "apply_patch", "delete_path", "git_commit", "http_request", "write_to_terminal"]) { + expect(canAlwaysAllow(tool)).toBe(false); + } + }); + + it("defaults to denying an unrecognised tool", () => { + expect(canAlwaysAllow("some_future_tool")).toBe(false); + }); +}); diff --git a/frontend/src/lib/tool-approval.ts b/frontend/src/lib/tool-approval.ts new file mode 100644 index 0000000..85bef6d --- /dev/null +++ b/frontend/src/lib/tool-approval.ts @@ -0,0 +1,40 @@ +// Which agent tools may be granted "always allow for this session". +// +// The bar for that grant is not "doesn't write to the workspace" — it's "the +// user has nothing to lose by never being asked again". Those differ: a tool +// can leave the workspace untouched and still reach the network or the disk. +// +// This matters because the model's inputs are not trusted. Agent mode reads +// file contents and web pages, and instructions embedded in that content can +// steer subsequent tool calls. The per-call approval prompt is what stops that +// from becoming unattended action, so anything with a side effect the user +// would want to see keeps its prompt. Notably that excludes: +// +// - web_search, fetch_url and the github_* tools, which each send a request +// to a destination the model chooses — a standing grant on those is an +// unattended outbound channel for whatever the model has already read. +// - capture_page_screenshot, which fetches a model-chosen URL *and* writes a +// PNG into the workspace, so it is not read-only in either sense. +// +// Listing what qualifies, rather than what doesn't, keeps this default-deny: a +// tool added to AGENT_TOOLS without being classified here needs per-call +// approval until someone decides otherwise. +export const AUTO_APPROVABLE_TOOLS = new Set([ + "read_file", + "find_files", + "file_info", + "list_dir", + "search_files", + "git_status", + "git_diff", + "git_log", + "read_notes", + "get_background_output", + "list_background_commands", + "find_symbol_references", + "read_terminal_output", +]); + +export function canAlwaysAllow(toolName: string): boolean { + return AUTO_APPROVABLE_TOOLS.has(toolName); +} diff --git a/frontend/src/lib/translations.ts b/frontend/src/lib/translations.ts index 0e03c9f..059e034 100644 --- a/frontend/src/lib/translations.ts +++ b/frontend/src/lib/translations.ts @@ -50,6 +50,7 @@ export interface Dictionary { yourSystem: string; cloudProviders: string; keysEncryptedNote: string; + keysNotEncryptedWarning: string; ollamaModelsSection: string; otherInstalledModels: string; chatDefaults: string; @@ -117,12 +118,28 @@ export interface Dictionary { mcpAdd: string; undoLastEdit: string; nothingToUndo: string; + terminalPanelToggle: string; + verificationCard: string; + verificationAttemptCounter: (attempt: number, max: number) => string; restoredFile: string; deletedNewFile: string; runTests: string; runLint: string; runFormat: string; newFile: string; + modelRuntime: string; + modelRuntimeHint: string; + modelRuntimeAutomatic: string; + modelRuntimeOllama: string; + modelRuntimeLlamaCpp: string; + modelRuntimeVllm: string; + modelRuntimeMlx: string; + ragEmbeddingModel: string; + ragEmbeddingModelHint: string; + ragCollections: string; + ragCollectionsHint: string; + ragCollectionsEmpty: string; + ragCollectionMeta: (docs: number, chunks: number) => string; modelsDir: string; modelsDirHint: string; modelsDirDefault: string; @@ -351,6 +368,25 @@ export interface Dictionary { profileBalancedNote: string; profileAutonomous: string; profileAutonomousNote: string; + sandboxSectionTitle: string; + sandboxSectionHint: string; + networkToolsLabel: string; + networkToolsHint: string; + sandboxMaxMemoryLabel: string; + sandboxMaxMemoryHint: string; + sandboxMaxCpuLabel: string; + sandboxMaxCpuHint: string; + sandboxStatusLabel: string; + sandboxStatusBubblewrap: string; + sandboxStatusSandboxExec: string; + sandboxStatusNone: string; + verificationSectionTitle: string; + verificationSectionHint: string; + verificationEnabledLabel: string; + verificationCommandsLabel: string; + verificationCommandsHint: string; + verificationMaxRetriesLabel: string; + verificationMaxRetriesHint: string; connectedAccountsTitle: string; connectedAccountsHint: string; accountGithubHint: string; @@ -415,6 +451,7 @@ export const en: Dictionary = { yourSystem: "Your system", cloudProviders: "Cloud providers", keysEncryptedNote: "Keys are encrypted at rest using your OS credential store and never leave this device.", + keysNotEncryptedWarning: "No OS credential store was found on this system, so keys are being saved in plain text on disk instead of encrypted. They still never leave this device, but anyone with file access to your user profile can read them.", ollamaModelsSection: "Ollama models", otherInstalledModels: "Other installed models", chatDefaults: "Chat defaults", @@ -483,12 +520,28 @@ export const en: Dictionary = { mcpAdd: "Add", undoLastEdit: "Undo last edit", nothingToUndo: "Nothing to undo.", + terminalPanelToggle: "Terminal", + verificationCard: "Verification", + verificationAttemptCounter: (attempt, max) => `attempt ${attempt}/${max}`, restoredFile: "Restored previous content of", deletedNewFile: "Removed newly-created file", runTests: "Run tests", runLint: "Lint", runFormat: "Format", newFile: "New file", + modelRuntime: "Model runtime", + modelRuntimeHint: "Which backend runs your models. Automatic picks per model based on its format (GGUF, safetensors, MLX) and your detected hardware (NVIDIA/AMD/Intel/Apple GPU, Apple Silicon) — GGUF goes to llama.cpp, safetensors on an NVIDIA/AMD GPU goes to vLLM, MLX-format models on Apple Silicon go to MLX, and anything else falls back to Transformers.", + modelRuntimeAutomatic: "Automatic (recommended)", + modelRuntimeOllama: "Ollama", + modelRuntimeLlamaCpp: "llama.cpp", + modelRuntimeVllm: "vLLM", + modelRuntimeMlx: "MLX", + ragEmbeddingModel: "RAG embedding model", + ragEmbeddingModelHint: "Used to index newly attached folders for retrieval. Existing collections keep whatever model they were created with — changing this only affects new ones.", + ragCollections: "Indexed collections", + ragCollectionsHint: "Folders you've attached for retrieval, persisted so re-attaching the same folder only re-embeds files that changed.", + ragCollectionsEmpty: "No folders indexed yet.", + ragCollectionMeta: (docs, chunks) => `${docs} document${docs === 1 ? "" : "s"} · ${chunks} chunk${chunks === 1 ? "" : "s"}`, modelsDir: "Model storage location", modelsDirHint: "Where Ollama downloads and stores model files. Changing this restarts Ollama (only if this app started it) — running downloads will be interrupted.", modelsDirDefault: "Default (Ollama's own location)", @@ -725,6 +778,25 @@ export const en: Dictionary = { profileBalancedNote: "Best default", profileAutonomous: "Autonomous", profileAutonomousNote: "Long coding tasks", + sandboxSectionTitle: "Sandbox & permissions", + sandboxSectionHint: "Controls what Agent mode's shell commands can reach — beyond the always-on blocklist for obviously destructive commands.", + networkToolsLabel: "Network-capable tools", + networkToolsHint: "Web search, page fetch/screenshot, HTTP requests, and the GitHub tools. Turning this off refuses those tool calls outright — the one restriction here that's fully enforced on every platform.", + sandboxMaxMemoryLabel: "Memory limit per command (MB)", + sandboxMaxMemoryHint: "A command that exceeds this is killed automatically. A safety net against a runaway process, not a strict quota — leave it generous.", + sandboxMaxCpuLabel: "CPU limit per command (%)", + sandboxMaxCpuHint: "Optional. Leave blank for no CPU limit.", + sandboxStatusLabel: "OS-level containment on this machine", + sandboxStatusBubblewrap: "Linux, via bubblewrap — commands run confined to the workspace, with network access denied unless a command explicitly requests it.", + sandboxStatusSandboxExec: "macOS, via sandbox-exec — commands run confined to the workspace, with network access denied unless a command explicitly requests it.", + sandboxStatusNone: "Not available on this platform — only the command blocklist and the resource limits below apply. On Linux, installing bubblewrap (bwrap) enables real filesystem and network containment.", + verificationSectionTitle: "Verification", + verificationSectionHint: "After the agent stops calling tools, optionally run real commands and feed the result back before treating the turn as done — off by default, since this runs commands automatically.", + verificationEnabledLabel: "Verify before finishing a turn", + verificationCommandsLabel: "Commands to run", + verificationCommandsHint: "One per line. Leave blank to use the workspace's detected build/test scripts automatically.", + verificationMaxRetriesLabel: "Max retries", + verificationMaxRetriesHint: "How many times the agent can try to fix a failing check before it stops and asks you.", connectedAccountsTitle: "Connected accounts", connectedAccountsHint: "Link developer services for repository analysis and access to private or gated models. Tokens stay encrypted locally when your OS credential store is available.", accountGithubHint: "Connect repositories for AI analysis and developer workflows.", @@ -790,6 +862,8 @@ export const tr: Dictionary = { cloudProviders: "Bulut sağlayıcılar", keysEncryptedNote: "Anahtarlar, işletim sistemi kimlik bilgisi deposu kullanılarak şifrelenir ve bu cihazdan çıkmaz.", + keysNotEncryptedWarning: + "Bu sistemde bir işletim sistemi kimlik bilgisi deposu bulunamadığı için anahtarlar şifrelenmeden düz metin olarak diske kaydediliyor. Yine de bu cihazdan çıkmazlar, ancak kullanıcı profilinize dosya erişimi olan herkes onları okuyabilir.", ollamaModelsSection: "Ollama modelleri", otherInstalledModels: "Diğer yüklü modeller", chatDefaults: "Sohbet varsayılanları", @@ -858,12 +932,28 @@ export const tr: Dictionary = { mcpAdd: "Ekle", undoLastEdit: "Son düzenlemeyi geri al", nothingToUndo: "Geri alınacak bir şey yok.", + terminalPanelToggle: "Terminal", + verificationCard: "Doğrulama", + verificationAttemptCounter: (attempt, max) => `${attempt}/${max}. deneme`, restoredFile: "Önceki içerik geri yüklendi:", deletedNewFile: "Yeni oluşturulan dosya kaldırıldı:", runTests: "Testleri çalıştır", runLint: "Lint", runFormat: "Biçimlendir", newFile: "Yeni dosya", + modelRuntime: "Model çalışma zamanı", + modelRuntimeHint: "Modellerinizi hangi arka ucun çalıştıracağı. Otomatik, her modeli formatına (GGUF, safetensors, MLX) ve algılanan donanımınıza (NVIDIA/AMD/Intel/Apple GPU, Apple Silicon) göre seçer — GGUF llama.cpp'ye, NVIDIA/AMD GPU'da safetensors vLLM'e, Apple Silicon'da MLX formatındaki modeller MLX'e gider; geri kalan her şey Transformers'a düşer.", + modelRuntimeAutomatic: "Otomatik (önerilen)", + modelRuntimeOllama: "Ollama", + modelRuntimeLlamaCpp: "llama.cpp", + modelRuntimeVllm: "vLLM", + modelRuntimeMlx: "MLX", + ragEmbeddingModel: "RAG gömme modeli", + ragEmbeddingModelHint: "Yeni eklenen klasörleri erişim için indekslemek üzere kullanılır. Mevcut koleksiyonlar oluşturuldukları modeli korur — bunu değiştirmek yalnızca yenilerini etkiler.", + ragCollections: "İndekslenmiş koleksiyonlar", + ragCollectionsHint: "Erişim için eklediğiniz klasörler, kalıcı olarak saklanır — aynı klasörü tekrar eklemek yalnızca değişen dosyaları yeniden gömer.", + ragCollectionsEmpty: "Henüz indekslenmiş klasör yok.", + ragCollectionMeta: (docs, chunks) => `${docs} belge · ${chunks} parça`, modelsDir: "Model depolama konumu", modelsDirHint: "Ollama'nın model dosyalarını indirdiği ve sakladığı yer. Bunu değiştirmek Ollama'yı yeniden başlatır (yalnızca bu uygulama başlattıysa) — devam eden indirmeler kesintiye uğrar.", modelsDirDefault: "Varsayılan (Ollama'nın kendi konumu)", @@ -1100,6 +1190,25 @@ export const tr: Dictionary = { profileBalancedNote: "En iyi varsayılan", profileAutonomous: "Özerk", profileAutonomousNote: "Uzun kodlama görevleri", + sandboxSectionTitle: "Korumalı alan ve izinler", + sandboxSectionHint: "Ajan modunun kabuk komutlarının nelere erişebileceğini kontrol eder — açıkça yıkıcı komutlar için her zaman etkin olan engelleme listesinin ötesinde.", + networkToolsLabel: "Ağ erişimi olan araçlar", + networkToolsHint: "Web arama, sayfa getirme/ekran görüntüsü, HTTP istekleri ve GitHub araçları. Bunu kapatmak bu araç çağrılarını tamamen reddeder — burada her platformda tam olarak uygulanan tek kısıtlama budur.", + sandboxMaxMemoryLabel: "Komut başına bellek sınırı (MB)", + sandboxMaxMemoryHint: "Bu sınırı aşan bir komut otomatik olarak sonlandırılır. Kesin bir kota değil, kaçak bir sürece karşı bir güvenlik önlemidir — cömert bırakın.", + sandboxMaxCpuLabel: "Komut başına CPU sınırı (%)", + sandboxMaxCpuHint: "İsteğe bağlı. CPU sınırı olmaması için boş bırakın.", + sandboxStatusLabel: "Bu makinede işletim sistemi düzeyinde sınırlama", + sandboxStatusBubblewrap: "Linux, bubblewrap aracılığıyla — komutlar çalışma alanıyla sınırlı çalışır, bir komut açıkça talep etmedikçe ağ erişimi reddedilir.", + sandboxStatusSandboxExec: "macOS, sandbox-exec aracılığıyla — komutlar çalışma alanıyla sınırlı çalışır, bir komut açıkça talep etmedikçe ağ erişimi reddedilir.", + sandboxStatusNone: "Bu platformda mevcut değil — yalnızca komut engelleme listesi ve aşağıdaki kaynak sınırları uygulanır. Linux'ta bubblewrap (bwrap) kurmak gerçek dosya sistemi ve ağ sınırlaması sağlar.", + verificationSectionTitle: "Doğrulama", + verificationSectionHint: "Ajan araç çağırmayı bıraktıktan sonra, isteğe bağlı olarak gerçek komutlar çalıştırıp sonucu geri bildirerek turu bitmiş saymadan önce kontrol edin — komutları otomatik çalıştırdığı için varsayılan olarak kapalıdır.", + verificationEnabledLabel: "Turu bitirmeden önce doğrula", + verificationCommandsLabel: "Çalıştırılacak komutlar", + verificationCommandsHint: "Her satıra bir tane. Çalışma alanının algılanan build/test betiklerini otomatik kullanmak için boş bırakın.", + verificationMaxRetriesLabel: "Maksimum yeniden deneme", + verificationMaxRetriesHint: "Ajanın başarısız bir kontrolü durup sizden yardım istemeden önce kaç kez düzeltmeyi deneyebileceği.", connectedAccountsTitle: "Bağlı hesaplar", connectedAccountsHint: "Depo analizi ve özel veya kısıtlı modellere erişim için geliştirici hizmetlerini bağlayın. İşletim sistemi kimlik bilgisi deposu kullanılabildiğinde jetonlar yerel olarak şifreli kalır.", accountGithubHint: "Yapay zeka analizi ve geliştirici iş akışları için depoları bağlayın.", diff --git a/frontend/src/lib/usage.test.ts b/frontend/src/lib/usage.test.ts index 1c79b72..44aa405 100644 --- a/frontend/src/lib/usage.test.ts +++ b/frontend/src/lib/usage.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { summarizeSession, aggregateBy } from "./usage"; +import { summarizeSession, aggregateBy, formatEnergy } from "./usage"; import type { ChatSession } from "@/types/electron"; function makeSession(overrides: Partial = {}): ChatSession { @@ -84,3 +84,9 @@ describe("aggregateBy", () => { expect(gpt4oMini?.[1].tokens).toBe(45); }); }); + +describe("formatEnergy", () => { + it("converts kWh to Wh using the correct factor", () => { + expect(formatEnergy(0.0005)).toBe("0.5 Wh"); + }); +}); diff --git a/frontend/src/lib/usage.ts b/frontend/src/lib/usage.ts index 2a4db0b..848d72c 100644 --- a/frontend/src/lib/usage.ts +++ b/frontend/src/lib/usage.ts @@ -44,6 +44,10 @@ export function summarizeSession(session: ChatSession): SessionUsage { }; } +export function formatEnergy(value: number): string { + return value < 0.001 ? `${(value * 1_000).toFixed(1)} Wh` : `${value.toFixed(4)} kWh`; +} + export function aggregateBy(usages: SessionUsage[], keyOf: (u: SessionUsage) => string): [string, UsageAggregate][] { const map = new Map(); for (const u of usages) { diff --git a/frontend/src/pages/Chat.tsx b/frontend/src/pages/Chat.tsx index 6086d17..d6bb670 100644 --- a/frontend/src/pages/Chat.tsx +++ b/frontend/src/pages/Chat.tsx @@ -17,12 +17,14 @@ import { FileDown, BookMarked, Database, + FileText, Loader2, Image as ImageIcon, Bot, Volume2, Mic, Undo2, + TerminalSquare, FlaskConical, Wand2, Wrench, @@ -58,6 +60,7 @@ import { } from "@/components/ui/dropdown-menu"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Markdown } from "@/components/markdown"; +import { TerminalPanel } from "@/components/terminal-panel"; import { cn } from "@/lib/utils"; import { useSessions } from "@/lib/sessions-context"; import { useI18n } from "@/lib/i18n"; @@ -70,6 +73,7 @@ import { speakText, stopSpeaking } from "@/lib/tts"; import { computeLineDiff } from "@/lib/diff"; import { useToast } from "@/components/toast"; import { isTransientError } from "@/lib/transient-errors"; +import { canAlwaysAllow } from "@/lib/tool-approval"; import { COMPACTION_BUDGET_TOKENS, COMPACTION_KEEP_RECENT, @@ -93,6 +97,7 @@ import type { PromptPreset, ProjectScripts, LocalGgufModel, + RagCitation, } from "@/types/electron"; type Attachment = AttachedFile & { folder?: string }; @@ -100,7 +105,7 @@ type Attachment = AttachedFile & { folder?: string }; interface RagFolder { folderPath: string; folderName: string; - indexId: string; + collectionId: string; chunkCount: number; } @@ -114,35 +119,12 @@ const RAG_THRESHOLD_CHARS = 20_000; // the most recent window renders by default, with older ones revealed a // window at a time on request rather than all at once. const RENDER_WINDOW_SIZE = 60; -// Read-only tools are safe to let the model call repeatedly without a fresh -// click each time — write_file and run_command always require explicit -// per-call approval since they have real, potentially irreversible effects. + interface PlanStep { text: string; done: boolean; } -const READ_ONLY_TOOLS = new Set([ - "read_file", - "find_files", - "file_info", - "list_dir", - "search_files", - "git_status", - "git_diff", - "git_log", - "web_search", - "fetch_url", - "read_notes", - "github_list_repositories", - "github_repository_tree", - "github_read_file", - "get_background_output", - "list_background_commands", - "capture_page_screenshot", - "find_symbol_references", -]); - // Vision models can already reason over any attached image — these just save // re-typing a good prompt for the common "I attached a diagram/wireframe" // case. Selecting one fills the composer; the user can still edit before sending. @@ -247,6 +229,24 @@ const MessageBubble = memo(function MessageBubble({ const { t } = useI18n(); if (m.role === "tool") { + if (m.isVerification) { + const passed = m.content.startsWith("Verification passed"); + return ( +
+
+
+ {passed ? : } {t.verificationCard} +
+ 
{m.content}
+
+
+ ); + } // Tool failures get a visually distinct card — an agent run that hit // an error mid-way should be scannable at a glance, not require // reading every result body to find where things went wrong. @@ -313,6 +313,21 @@ const MessageBubble = memo(function MessageBubble({ )}
)} + {m.citations && m.citations.length > 0 && ( +
+ {m.citations.map((c, citeIdx) => ( + + + {c.name} + {c.page ? `p.${c.page}` : `L${c.startLine}-${c.endLine}`} + + ))} +
+ )}
+ )} {agentMode && agentWorkspace && projectScripts.test && ( - {READ_ONLY_TOOLS.has(call.name) && ( + {canAlwaysAllow(call.name) && ( + +

Download Center

Persistent model transfers, verification, and recovery

+
+ {recovery.recoveredAt &&
Startup recovery: {recovery.recoveredJobs} job{recovery.recoveredJobs === 1 ? "" : "s"}
} + + +
+
Overall progress{bytes(totals.received)} / {bytes(totals.total)} · {overallPercent.toFixed(1)}%
+
+ + + +
+
+ + {jobs.length === 0 ?

No downloads yet

Model downloads started from Settings will appear here and survive restarts.

: +
{jobs.map((job) => { + const received = job.shards.reduce((sum, shard) => sum + shard.receivedBytes, 0); + const total = job.shards.reduce((sum, shard) => sum + shard.expectedBytes, 0); + const percent = total ? received / total * 100 : 0; const forecast = forecasts[job.id]; + return
+
+

{job.modelName}

{stateLabel(job)}{job.recoveredAtStartup && Recovered}

{job.publisher} · {job.backend}{job.quantization ? ` · ${job.quantization}` : ""}

+
+ {["queued", "resolving", "downloading"].includes(job.state) && } + {job.state === "paused" && } + {job.state === "failed" && } + {!["ready", "cancelled"].includes(job.state) && } + +
+
+
{bytes(received)} / {bytes(total)} ({percent.toFixed(1)}%)Speed: {job.bytesPerSecond ? `${bytes(job.bytesPerSecond)}/s` : "—"}ETA: {duration(job.etaSeconds)}{forecast?.availableBytes == null ? "Disk unknown" : `${bytes(forecast.availableBytes)} free`}
+ {job.error &&

{job.error.message}

} +
Shards ({job.shards.length})
{job.shards.map((shard) => { const shardPercent = shard.expectedBytes ? shard.receivedBytes / shard.expectedBytes * 100 : 0; return
{shard.filename}{shard.state === "verifying" ? "Verifying" : shard.state === "ready" ? <>Verified : shard.receivedBytes > 0 ? ".part" : shard.state} · {bytes(shard.receivedBytes)} / {bytes(shard.expectedBytes)}
; })}
+
; + })}
} +
+ ; +} diff --git a/frontend/src/pages/RuntimeManager.tsx b/frontend/src/pages/RuntimeManager.tsx new file mode 100644 index 0000000..c7fcabf --- /dev/null +++ b/frontend/src/pages/RuntimeManager.tsx @@ -0,0 +1,82 @@ +import { useCallback, useEffect, useState, type ReactElement } from "react"; +import { Activity, ArrowLeft, CircleStop, Clipboard, Cpu, HardDrive, PackageOpen, Play, RefreshCw, RotateCw, Server, Terminal, TriangleAlert, Wrench, XCircle } from "lucide-react"; +import { useNavigate } from "react-router-dom"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import type { LocalRuntimeStatus, PythonEnvironmentStatus } from "@/types/electron"; + +const backendNames = { mlx: "MLX", rocm: "ROCm llama.cpp", vllm: "vLLM" } as const; +const stateStyles = { + starting: "bg-blue-500/10 text-blue-600", running: "bg-emerald-500/10 text-emerald-600", + unhealthy: "bg-destructive/10 text-destructive", stopped: "bg-muted text-muted-foreground", +}; +function memory(value: number | null): string { return value === null ? "Unavailable" : value >= 1024 ? `${(value / 1024).toFixed(1)} GB` : `${value.toFixed(0)} MB`; } +function uptime(seconds: number): string { const h = Math.floor(seconds / 3600); const m = Math.floor(seconds % 3600 / 60); const s = Math.floor(seconds % 60); return h ? `${h}h ${m}m` : m ? `${m}m ${s}s` : `${s}s`; } + +export default function RuntimeManager() { + const navigate = useNavigate(); + const [statuses, setStatuses] = useState([]); + const [models, setModels] = useState>({}); + const [busy, setBusy] = useState>({}); + const [expandedLogs, setExpandedLogs] = useState>({}); + const [error, setError] = useState>({}); + const [environments, setEnvironments] = useState([]); + const [environmentsLoading, setEnvironmentsLoading] = useState(false); + const hasApi = typeof window !== "undefined" && !!window.api?.localBackends; + // Process status (PID/port/uptime/logs) is cheap to poll — it just reads + // already-running child processes. Python environment inspection is not: + // per family it can spawn nvidia-smi/rocminfo/wsl.exe and a Python + // interpreter just to check installed package versions, each with its own + // multi-second timeout. Polling that every 2.5s alongside process status + // was a subprocess storm for state that only changes on install/repair, so + // it's fetched once on entry and re-fetched only on those explicit triggers + // (or a manual refresh) rather than on the same interval as process status. + const refreshStatuses = useCallback(async () => { if (hasApi) setStatuses(await window.api.localBackends.getStatuses()); }, [hasApi]); + const refreshEnvironments = useCallback(async () => { + if (!hasApi) return; + setEnvironmentsLoading(true); + try { setEnvironments(await window.api.pythonRuntimes.getStatuses()); } + finally { setEnvironmentsLoading(false); } + }, [hasApi]); + const refreshAll = useCallback(async () => { await Promise.all([refreshStatuses(), refreshEnvironments()]); }, [refreshStatuses, refreshEnvironments]); + useEffect(() => { void refreshStatuses(); const timer = setInterval(() => void refreshStatuses(), 2500); return () => clearInterval(timer); }, [refreshStatuses]); + useEffect(() => { void refreshEnvironments(); }, [refreshEnvironments]); + useEffect(() => { setModels((current) => Object.fromEntries(statuses.map((status) => [status.backend, current[status.backend] ?? status.model ?? ""]))); }, [statuses]); + + async function run(backend: LocalRuntimeStatus["backend"], action: "start" | "stop" | "restart" | "unload") { + setBusy((value) => ({ ...value, [backend]: true })); setError((value) => ({ ...value, [backend]: "" })); + try { + if (action === "start" || action === "restart") { + const model = models[backend]?.trim(); if (!model) throw new Error("Enter a Hugging Face model ID or local model path first."); + await window.api.localBackends[action](backend, model); + } else await window.api.localBackends[action](backend); + } catch (reason) { setError((value) => ({ ...value, [backend]: (reason as Error).message })); } + finally { setBusy((value) => ({ ...value, [backend]: false })); await refreshStatuses(); } + } + + return
+

Runtime Manager

Managed MLX, ROCm, and vLLM processes

+

Managed Python environments

Every runtime family has an isolated, pinned virtual environment. Commands are shown for your approval and are never executed silently.

{environments.map((environment) =>

{environment.family}

{environment.state}

{environment.destination}

Disk required
{memory(environment.manifest.diskRequirementBytes / 1024 / 1024)}
Expected download
{memory(environment.manifest.expectedDownloadBytes / 1024 / 1024)}
Python
{environment.pythonVersion ?? environment.manifest.python}
Worker protocol
v{environment.manifest.protocolVersion}

{Object.entries(environment.manifest.packages).map(([name, version]) => `${name}==${version}`).join(" · ")}

{environment.issues.length > 0 &&

{environment.issues.join(" ")}

}
{environment.state === "missing" && }{environment.state === "drifted" && }
{environment.state !== "healthy" && environment.state !== "incompatible" &&
Review command
{environment.state === "drifted" ? environment.repairCommand : environment.installCommand}
}
)}
+
{statuses.map((status) => { + const model = models[status.backend] ?? ""; const logsOpen = expandedLogs[status.backend]; const runtimeError = error[status.backend] || status.startupError; + return
+

{backendNames[status.backend]}

{status.state}{status.installed ? "Installed" : "Not installed"}{status.compatible ? "Compatible" : "Incompatible"}

{status.detail}

+
{status.state === "stopped" ? : <>}
+ +
+ } label="PID" value={status.pid?.toString() ?? "—"} />} label="Port" value={status.port?.toString() ?? "Dynamic"} />} label="Uptime" value={status.running ? uptime(status.uptimeSeconds) : "—"} />} label="RAM" value={memory(status.ramMB)} />} label="VRAM" value={memory(status.vramMB)} />} label="GPU device" value={status.device ?? "Unavailable"} /> +
+ {status.environmentIssues.length > 0 &&
{status.environmentIssues.join(" · ")}
} + {runtimeError &&
{runtimeError}
} + {!status.installed && status.compatible &&

Installation guidance

{status.installCommand}
} +
+
{logsOpen && 
{status.logs.length ? status.logs.join("\n") : "No runtime output captured yet."}
}
+
; + })}
+ {!hasApi &&

Runtime management is available in the desktop application.

} +
; +} + +function Metric({ icon, label, value }: { icon: ReactElement; label: string; value: string }) { + return
{icon}{label}

{value}

; +} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index a9684bc..ec8128e 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -32,6 +32,8 @@ import { Apple, Gauge, ExternalLink, + Shield, + ShieldAlert, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; @@ -68,8 +70,11 @@ import type { AppActivity, LinkedAccount, LocalRuntimeStatus, + SandboxCapabilities, + BenchmarkResult, + RagCollectionSummary, } from "@/types/electron"; -import { EXTRA_MODELS } from "@/lib/model-catalog"; +import { EXTRA_MODELS, EMBEDDING_MODELS } from "@/lib/model-catalog"; import { recommendGpuBackend, gpuBackendNote } from "@/lib/gpu"; import { useToast } from "@/components/toast"; import { @@ -80,7 +85,7 @@ import { notifyKeybindingsChanged, type KeybindingAction, } from "@/lib/keybindings"; -import { OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS, formatModelRef, CUSTOM_PROVIDER_PRESETS } from "@/lib/providers"; +import { OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS, formatModelRef, parseModelRef, CUSTOM_PROVIDER_PRESETS } from "@/lib/providers"; import { useSessions } from "@/lib/sessions-context"; import { useI18n } from "@/lib/i18n"; import type { Locale } from "@/lib/translations"; @@ -156,6 +161,7 @@ export default function Settings() { const [running, setRunning] = useState(null); const [specs, setSpecs] = useState(null); const [recommendations, setRecommendations] = useState(null); + const [ragCollections, setRagCollections] = useState([]); const [installed, setInstalled] = useState([]); const [pulling, setPulling] = useState>({}); const [settings, setSettings] = useState(null); @@ -186,6 +192,16 @@ export default function Settings() { const [userDataPath, setUserDataPath] = useState(null); const [activity, setActivity] = useState(null); const [activityLoading, setActivityLoading] = useState(false); + const [benchmarkModel, setBenchmarkModel] = useState(""); + const [benchmarkContext, setBenchmarkContext] = useState(8192); + const [benchmarkCompare, setBenchmarkCompare] = useState(true); + const [benchmarkRunning, setBenchmarkRunning] = useState(false); + const [benchmarkRequestId, setBenchmarkRequestId] = useState(null); + const [benchmarkResult, setBenchmarkResult] = useState(null); + const [tariffName, setTariffName] = useState(""); + const [tariffStartHour, setTariffStartHour] = useState(17); + const [tariffEndHour, setTariffEndHour] = useState(22); + const [tariffPrice, setTariffPrice] = useState(0.3); const [keybindings, setKeybindings] = useState>(DEFAULT_KEYBINDINGS); const [localModelInput, setLocalModelInput] = useState(""); const [localModelBackend, setLocalModelBackend] = useState<"mlx" | "vllm">("vllm"); @@ -195,6 +211,8 @@ export default function Settings() { const [keybindingConflict, setKeybindingConflict] = useState(null); const [importMessage, setImportMessage] = useState(null); const [ollamaHostInput, setOllamaHostInput] = useState(""); + const [sandboxCapabilities, setSandboxCapabilities] = useState(null); + const [secretsEncrypted, setSecretsEncrypted] = useState(null); const [modelsDirStatus, setModelsDirStatus] = useState(null); const [changingModelsDir, setChangingModelsDir] = useState(false); const [newPresetName, setNewPresetName] = useState(""); @@ -207,6 +225,10 @@ export default function Settings() { const activePullCount = useRef(0); const loadedTabs = useRef(new Set()); + useEffect(() => { + window.api.benchmark.getLast().then(setBenchmarkResult).catch(() => undefined); + }, []); + const [mcpStatuses, setMcpStatuses] = useState>({}); const [mcpConnecting, setMcpConnecting] = useState>({}); const [showAddMcp, setShowAddMcp] = useState(false); @@ -250,6 +272,7 @@ export default function Settings() { window.api.ollama.status().then(setRunning); window.api.system.getSpecs().then(setSpecs); window.api.system.getRecommendations().then(setRecommendations); + window.api.rag.listCollections().then(setRagCollections); window.api.settings.get().then((s) => { setSettings(s); setOllamaHostInput(s.ollamaHost); @@ -259,6 +282,8 @@ export default function Settings() { window.api.llamacpp.listModels().then(setLlamaCppModels); window.api.llamacpp.getAvailableGpuBackends().then(setLlamaCppGpuBackends); refreshRuntimeStatuses(); + window.api.agent.getSandboxCapabilities().then(setSandboxCapabilities); + window.api.secrets.isEncryptionAvailable().then(setSecretsEncrypted); }, []); useEffect(() => { @@ -324,21 +349,16 @@ export default function Settings() { setHfFilesLoading(false); } - async function downloadForLlamaCpp(modelId: string, filename: string) { + async function downloadForLlamaCpp(modelId: string, filename: string, expectedBytes: number | null) { const key = `${modelId}/${filename}`; setHfDownloading((d) => ({ ...d, [key]: 0 })); - const res = await window.api.huggingface.downloadFile(modelId, filename, (progress) => { - if (progress.totalBytes) { - setHfDownloading((d) => ({ ...d, [key]: Math.round((progress.receivedBytes / progress.totalBytes!) * 100) })); - } - }); + await window.api.downloads.create({ modelId, filename, expectedBytes: expectedBytes ?? 0 }); setHfDownloading((d) => { const next = { ...d }; delete next[key]; return next; }); - if (res.error) toast.error(res.error); - else window.api.llamacpp.listModels().then(setLlamaCppModels); + toast.success("Added to Download Center"); } async function deleteLlamaCppModel(name: string) { @@ -492,6 +512,66 @@ export default function Settings() { setTimeout(() => setDiagnosticsCopied(false), 1500); } + async function runHardwareBenchmark() { + const parsed = parseModelRef(benchmarkModel); + if (!parsed) { + toast.error("Select a local model to benchmark."); + return; + } + setBenchmarkRunning(true); + const { requestId, promise } = window.api.benchmark.run({ + provider: parsed.provider, + model: parsed.modelId, + maxContextLength: benchmarkContext, + outputTokens: 96, + compareCpuGpu: benchmarkCompare, + }); + setBenchmarkRequestId(requestId); + try { + const response = await promise; + if (response.result) { + setBenchmarkResult(response.result); + toast.success("Benchmark completed."); + } else if (response.error) { + toast.error(response.error); + } + } finally { + setBenchmarkRunning(false); + setBenchmarkRequestId(null); + } + } + + async function cancelHardwareBenchmark() { + if (benchmarkRequestId) await window.api.benchmark.cancel(benchmarkRequestId); + } + + async function exportHardwareDiagnostic() { + if (!benchmarkResult) return; + const result = await window.api.benchmark.exportReport(benchmarkResult); + if (result.success) toast.success("Diagnostic report exported."); + } + + async function addTimeOfUseTariff() { + if (!settings || !tariffName.trim() || tariffPrice < 0) return; + await saveSettings({ + timeOfUseTariffs: [ + ...(settings.timeOfUseTariffs ?? []), + { + name: tariffName.trim(), + startHour: Math.max(0, Math.min(24, tariffStartHour)), + endHour: Math.max(0, Math.min(24, tariffEndHour)), + pricePerKwh: tariffPrice, + }, + ], + }); + setTariffName(""); + } + + async function removeTimeOfUseTariff(index: number) { + if (!settings) return; + await saveSettings({ timeOfUseTariffs: (settings.timeOfUseTariffs ?? []).filter((_, itemIndex) => itemIndex !== index) }); + } + async function saveOpenaiKey() { const value = openaiKeyInput.trim(); await window.api.secrets.set("openai_api_key", value); @@ -680,6 +760,11 @@ export default function Settings() { refreshInstalled(); } + async function deleteRagCollection(id: string) { + await window.api.rag.deleteCollection(id); + setRagCollections((prev) => prev.filter((c) => c.collectionId !== id)); + } + async function saveSettings(partial: Partial) { if (!settings) return; const merged = { ...settings, ...partial }; @@ -1010,6 +1095,66 @@ export default function Settings() { {modelsDirStatus &&

{modelsDirStatus}

} + + + + + + + + + + {ragCollections.length === 0 ? ( +

{t.ragCollectionsEmpty}

+ ) : ( +
+ {ragCollections.map((c) => ( +
+
+
{c.name}
+
+ {c.folderPath} · {t.ragCollectionMeta(c.documentCount, c.chunkCount)} · {c.embeddingModel} +
+
+ +
+ ))} +
+ )} {settings && ( @@ -1481,11 +1626,16 @@ export default function Settings() { 🔧 Tool calling )} - {m.runsOnGpu && Runs on GPU} - {!m.fits && May be too large} + {m.outcome}

{m.description}

-

Needs ~{m.minRAMGB} GB RAM

+

{m.reason}

+

+ {m.quantization} · weights {m.estimatedWeightGB} GB · KV cache {m.estimatedKvCacheGB} GB · overhead {m.runtimeOverheadGB} GB · {m.expectedGpuOffloadPercent}% GPU offload +

+

+ Runtime: {m.recommendedRuntime} · estimated {m.estimatedTokensPerSecond} tok/s{m.measuredTokensPerSecond !== undefined ? ` · measured ${m.measuredTokensPerSecond} tok/s` : ""} +

{isInstalled ? ( @@ -1640,7 +1790,15 @@ export default function Settings() {
- + + {secretsEncrypted === false && ( + +
+ + {t.keysNotEncryptedWarning} +
+ + )}
{openaiKeySet && ( @@ -2450,6 +2608,106 @@ export default function Settings() { + + + + + + saveSettings({ sandboxMaxMemoryMB: Math.max(128, Number(e.target.value)) })} + className="w-28" + aria-label={t.sandboxMaxMemoryLabel} + /> + + + + saveSettings({ + sandboxMaxCpuPercent: e.target.value === "" ? undefined : Math.max(1, Math.min(100, Number(e.target.value))), + }) + } + className="w-28" + aria-label={t.sandboxMaxCpuLabel} + /> + + +
+ {sandboxCapabilities?.mechanism === "bubblewrap" || sandboxCapabilities?.mechanism === "sandbox-exec" ? ( + + ) : ( + + )} + + {!sandboxCapabilities + ? "…" + : sandboxCapabilities.mechanism === "bubblewrap" + ? t.sandboxStatusBubblewrap + : sandboxCapabilities.mechanism === "sandbox-exec" + ? t.sandboxStatusSandboxExec + : t.sandboxStatusNone} + +
+ + + + + + + + +