diff --git a/README.md b/README.md index 03fc399473e..5967bfa7c12 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ Manage your install with `bun run sim`: ```bash bun run sim start | stop | restart # bring your install up / down / cycle +bun run sim update # pull/rebuild and apply Compose images bun run sim status # what's installed and healthy bun run sim logs # follow logs bun run sim doctor # diagnose configuration problems diff --git a/apps/docs/content/docs/en/platform/self-hosting/docker.mdx b/apps/docs/content/docs/en/platform/self-hosting/docker.mdx index 640a18d6448..1b911191227 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/docker.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/docker.mdx @@ -149,8 +149,8 @@ docker compose -f docker-compose.prod.yml logs migrations # Is the scheduler firing? docker compose -f docker-compose.prod.yml logs -f cron -# Upgrade: bump SIM_VERSION in .env, then -docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d +# Upgrade: bump SIM_VERSION in .env when pinned, then +bun run sim update ``` backup.sql. Restore with: docker compose -f docker-compose.prod.yml exec -T db psql -U postgres simstudio < backup.sql. The database data is persisted in a Docker volume named postgres_data."}, { question: "Can I customize the PostgreSQL credentials?", answer: "Yes. The docker-compose.prod.yml uses environment variable defaults: POSTGRES_USER (default: postgres), POSTGRES_PASSWORD (default: postgres), POSTGRES_DB (default: simstudio), and POSTGRES_PORT (default: 5432). Set these in your .env file to override them." }, ]} /> - diff --git a/apps/docs/content/docs/en/platform/self-hosting/upgrades.mdx b/apps/docs/content/docs/en/platform/self-hosting/upgrades.mdx index 69b83291f99..265d56e53b9 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/upgrades.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/upgrades.mdx @@ -142,11 +142,14 @@ kubectl logs -n simstudio deploy/sim-app -c migrations --tail=100 ```bash -docker compose -f docker-compose.prod.yml pull -docker compose -f docker-compose.prod.yml up -d +bun run sim update docker compose -f docker-compose.prod.yml logs migrations ``` +`bun run sim update` pulls the versions configured by `SIM_VERSION` (or `latest` when it is +unset), recreates the changed services, and keeps data volumes. It is equivalent to running +`docker compose pull` followed by `docker compose up -d`. + There is a short window where the app is unavailable while containers restart. Compose has no rolling-update mechanism — plan a maintenance window, or run Kubernetes if you need zero-downtime upgrades. diff --git a/scripts/setup/capability-config.test.ts b/scripts/setup/capability-config.test.ts index 9e9a37f8d99..de03288280e 100644 --- a/scripts/setup/capability-config.test.ts +++ b/scripts/setup/capability-config.test.ts @@ -10,6 +10,7 @@ import { defineCapabilitySetup, EMAIL_SETUP, getOAuthClientSetupFields, + KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP, } from './capability-config.ts' import { getCapabilitySetupOptions } from './capability-setup.ts' @@ -76,6 +77,12 @@ describe('capability setup configuration', () => { ) }) + it('offers OpenAI first for fresh knowledge embedding setup', () => { + expect( + getCapabilitySetupOptions(KNOWLEDGE_EMBEDDINGS_SETUP).map((option) => option.id) + ).toEqual(['openai', 'azure-openai', 'openrouter']) + }) + it('maps every OAuth runtime field to a CLI input mode in runtime order', () => { for (const id of Object.keys(OAUTH_CLIENT_CAPABILITIES) as Array< keyof typeof OAUTH_CLIENT_CAPABILITIES diff --git a/scripts/setup/capability-config.ts b/scripts/setup/capability-config.ts index 923f7e00c02..486d15d6be7 100644 --- a/scripts/setup/capability-config.ts +++ b/scripts/setup/capability-config.ts @@ -823,7 +823,7 @@ export const KNOWLEDGE_EMBEDDINGS_SETUP = defineCapabilitySetup(KNOWLEDGE_EMBEDD ], }, }, - optionOrder: ['azure-openai', 'openai', 'openrouter'], + optionOrder: ['openai', 'azure-openai', 'openrouter'], }) export const CAPABILITY_SETUPS = [ diff --git a/scripts/setup/capability-setup.ts b/scripts/setup/capability-setup.ts index bad7fa26e43..413f9d7f914 100644 --- a/scripts/setup/capability-setup.ts +++ b/scripts/setup/capability-setup.ts @@ -49,6 +49,8 @@ interface PromptState { values: Record } +const SKIP_OPTION_ID = '__skip-capability-setup__' + /** Stages a capability transition into a larger setup run without losing prompt context. */ export function stageCapabilitySetupTransition( currentValues: Map, @@ -176,11 +178,11 @@ export function getCapabilitySetupOptions( }) } -/** Resolves the setup option representing the effective current configuration. */ +/** Resolves the setup option representing the effective current configuration, if one exists. */ export function resolveCurrentCapabilitySetupOptionId( setup: CapabilitySetupDefinition, values: EnvCapabilityValues -): string { +): string | undefined { const options = getCapabilitySetupOptions(setup) const explicitAction = options.find( (option) => @@ -214,9 +216,10 @@ export function resolveCurrentCapabilitySetupOptionId( const firstAction = options.find((option) => option.kind === 'action') if (firstAction) return firstAction.id - throw new Error( - `Capability ${setup.definition.id} has no setup option for its current configuration` - ) + if (options.length === 0) { + throw new Error(`Capability ${setup.definition.id} has no setup options`) + } + return undefined } /** Applies selector and activation inference to the CLI-entered values. */ @@ -473,6 +476,21 @@ async function renderPrompts(prompts: readonly SetupPrompt[], state: PromptState } } +async function promptSelectedCapabilitySetup( + setup: CapabilitySetupDefinition, + selected: ResolvedSetupOption, + currentValues: ReadonlyMap +): Promise { + const state: PromptState = { + setup, + optionId: selected.id, + currentValues, + values: {}, + } + await renderPrompts(selected.prompts, state) + return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues) +} + /** Renders a CLI-owned capability setup and returns its validated environment transition. */ export async function promptCapabilitySetup( setup: CapabilitySetupDefinition, @@ -497,12 +515,44 @@ export async function promptCapabilitySetup( ) } - const state: PromptState = { - setup, - optionId: selected.id, - currentValues, - values: {}, + return promptSelectedCapabilitySetup(setup, selected, currentValues) +} + +/** Offers capability providers while allowing the user to leave configuration unchanged. */ +export async function promptOptionalCapabilitySetup( + setup: CapabilitySetupDefinition, + currentValues: ReadonlyMap, + context: CapabilitySetupContext, + skipHint: string +): Promise { + const options = getCapabilitySetupOptions(setup) + if (options.some((option) => option.id === SKIP_OPTION_ID)) { + throw new Error(`Capability ${setup.definition.id} uses reserved option ${SKIP_OPTION_ID}`) } - await renderPrompts(selected.prompts, state) - return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues) + const currentOptionId = resolveCurrentCapabilitySetupOptionId(setup, currentValues) + const selectedOptionId = await p.select({ + message: setup.message, + options: [ + ...options.map((option) => ({ + value: option.id, + label: option.label, + hint: markCurrentlyUsed(resolveHint(option.hint, context), option.id === currentOptionId), + })), + { + value: SKIP_OPTION_ID, + label: 'Not now', + hint: currentOptionId ? 'leave the current configuration unchanged' : skipHint, + }, + ], + initialValue: currentOptionId ?? options[0]?.id, + }) + if (selectedOptionId === SKIP_OPTION_ID) return null + + const selected = options.find((option) => option.id === selectedOptionId) + if (!selected) { + throw new Error( + `Capability ${setup.definition.id} returned unknown setup option ${selectedOptionId}` + ) + } + return promptSelectedCapabilitySetup(setup, selected, currentValues) } diff --git a/scripts/setup/index.ts b/scripts/setup/index.ts index cc5660e3da1..d60d6bf8634 100755 --- a/scripts/setup/index.ts +++ b/scripts/setup/index.ts @@ -18,6 +18,7 @@ const USAGE = `Usage: bun run sim setup configure one feature bun run sim doctor [--fix] [--json] check your setup bun run sim start | stop | restart bring your install up / down / cycle + bun run sim update pull/rebuild and apply Compose images bun run sim status what's installed and healthy bun run sim logs follow logs bun run sim down remove containers (data kept) diff --git a/scripts/setup/lifecycle.test.ts b/scripts/setup/lifecycle.test.ts new file mode 100644 index 00000000000..76e7a942b74 --- /dev/null +++ b/scripts/setup/lifecycle.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from 'bun:test' +import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle.ts' + +describe('setup lifecycle', () => { + it('recognizes update as a lifecycle command', () => { + expect(isLifecycleCommand('update')).toBe(true) + }) + + it('pulls published installs and rebuilds source installs', () => { + expect(getComposeUpdateMode('/repo/docker-compose.prod.yml')).toBe('pull') + expect(getComposeUpdateMode('/repo/docker-compose.local.yml')).toBe('build') + expect(() => getComposeUpdateMode('/repo/compose.yml')).toThrow(/Unsupported Sim Compose file/) + }) +}) diff --git a/scripts/setup/lifecycle.ts b/scripts/setup/lifecycle.ts index 3ed94657f9d..e93c43d9b57 100644 --- a/scripts/setup/lifecycle.ts +++ b/scripts/setup/lifecycle.ts @@ -20,6 +20,7 @@ export const LIFECYCLE_COMMANDS = [ 'start', 'stop', 'restart', + 'update', 'status', 'logs', 'down', @@ -325,6 +326,50 @@ function restart(install: Install): void { p.note(k8sReachHints(install.context), 'Kubernetes is managed with kubectl') } +export type ComposeUpdateMode = 'pull' | 'build' + +/** Resolves how a setup-managed Compose install obtains its next image. */ +export function getComposeUpdateMode(file: string): ComposeUpdateMode { + const name = path.basename(file) + if (name === 'docker-compose.prod.yml') return 'pull' + if (name === 'docker-compose.local.yml') return 'build' + throw new Error(`Unsupported Sim Compose file: ${file}`) +} + +function update(install: Install): void { + if (install.kind === 'dev') { + throw new SetupError('sim update is only available for Docker Compose installs.', [ + 'update the source checkout with git, run bun install, then restart bun run dev:full', + ]) + } + if (install.kind === 'k8s') { + throw new SetupError('sim update does not upgrade Kubernetes releases.', [ + 'upgrade the release with helm after reviewing the chart and release notes', + ]) + } + + const mode = getComposeUpdateMode(install.file) + const spin = p.spinner() + if (mode === 'pull') { + spin.start('Pulling configured Sim images…') + dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir) + } else { + spin.start('Rebuilding Sim images with current base images…') + dockerRun(composeArgs(install, 'build', '--pull'), 'docker compose build failed', install.dir) + } + spin.message('Applying updated images and running migrations…') + dockerRun(composeArgs(install, 'up', '-d'), 'docker compose up failed', install.dir) + spin.stop('Sim updated (data volumes kept)') + p.note( + [ + `version: ${theme.command(`SIM_VERSION in ${path.join(install.dir, '.env')}`)} (latest when unset)`, + `check: ${theme.command('bun run sim status')}`, + `logs: ${theme.command('bun run sim logs')}`, + ].join('\n'), + 'Update complete' + ) +} + function showLogs(install: Install): void { if (install.kind === 'compose') { dockerInherit(composeArgs(install, 'logs', '-f', '--tail', '100'), install.dir) @@ -488,6 +533,8 @@ export async function runLifecycle(command: LifecycleCommand): Promise { return stop(install) case 'restart': return restart(install) + case 'update': + return update(install) case 'logs': return showLogs(install) case 'down': diff --git a/scripts/setup/modes/compose.ts b/scripts/setup/modes/compose.ts index aa1be4ed3bd..39fb7d1cd02 100644 --- a/scripts/setup/modes/compose.ts +++ b/scripts/setup/modes/compose.ts @@ -13,6 +13,7 @@ import { collectSecrets, mothershipOverride, promptCopilotKey, + promptKnowledgeEmbeddings, promptLlmKeys, promptSecurity, promptSignInProviders, @@ -117,9 +118,13 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom if (copilotKey) values.COPILOT_API_KEY = copilotKey Object.assign(values, chatFlagValues(copilotKey)) Object.assign(values, await promptLlmKeys(detection, !quick)) + const stagedVars = new Map(root.vars) + for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) + const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true }) + if (embeddings) { + stageCapabilitySetupTransition(stagedVars, values, remove, embeddings) + } if (!quick) { - const stagedVars = new Map(root.vars) - for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) const storage = await promptCapabilitySetup(STORAGE_SETUP, stagedVars, { containerized: true, }) diff --git a/scripts/setup/modes/dev.ts b/scripts/setup/modes/dev.ts index 388adc98149..86d44a88023 100644 --- a/scripts/setup/modes/dev.ts +++ b/scripts/setup/modes/dev.ts @@ -15,6 +15,7 @@ import { collectSecrets, mothershipOverride, promptCopilotKey, + promptKnowledgeEmbeddings, promptLlmKeys, promptSecurity, promptSignInProviders, @@ -120,9 +121,14 @@ export async function runDevMode( writeEnvValues('realtime', { REDIS_URL: redisUrl }) } + const stagedVars = new Map(simAfter.vars) + for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) + const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: false }) + if (embeddings) { + stageCapabilitySetupTransition(stagedVars, values, remove, embeddings) + } + if (!quick) { - const stagedVars = new Map(simAfter.vars) - for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) for (const setup of [JOBS_SETUP, STORAGE_SETUP, EMAIL_SETUP] as const) { const transition = await promptCapabilitySetup(setup, stagedVars, { containerized: false, diff --git a/scripts/setup/modes/k8s.ts b/scripts/setup/modes/k8s.ts index 68a2d919bec..b4261e37dad 100644 --- a/scripts/setup/modes/k8s.ts +++ b/scripts/setup/modes/k8s.ts @@ -1,12 +1,19 @@ import { spawn, spawnSync } from 'node:child_process' import { getErrorMessage } from '@sim/utils/errors' +import { KNOWLEDGE_EMBEDDINGS_SETUP } from '../capability-config.ts' +import { getCapabilitySetupFields, stageCapabilitySetupTransition } from '../capability-setup.ts' import type { Detection } from '../detect.ts' import { ensureDocker } from '../docker.ts' import { generateSecret, ROOT } from '../env-files.ts' import { SetupError } from '../errors.ts' import { waitFor } from '../probes.ts' import * as p from '../prompter.ts' -import { chatFlagValues, mothershipOverride, promptCopilotKey } from '../steps.ts' +import { + chatFlagValues, + mothershipOverride, + promptCopilotKey, + promptKnowledgeEmbeddings, +} from '../steps.ts' import { glyph, theme } from '../theme.ts' import { APP_SIGNUP_URL, APP_URL } from '../urls.ts' @@ -362,6 +369,17 @@ export async function runK8sMode(detection: Detection): Promise { ...(copilotKey ? { COPILOT_API_KEY: copilotKey } : {}), ...chatFlagValues(copilotKey), } + const stagedVars = new Map(Object.entries(releaseValues?.app?.env ?? {})) + for (const [key, value] of Object.entries(appEnv)) stagedVars.set(key, value) + for (const key of getCapabilitySetupFields(KNOWLEDGE_EMBEDDINGS_SETUP)) { + const existing = stagedVars.get(key) + if (existing) appEnv[key] = existing + } + const remove = new Set() + const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true }) + if (embeddings) { + stageCapabilitySetupTransition(stagedVars, appEnv, remove, embeddings) + } const spin = p.spinner() spin.start('helm upgrade --install (first run pulls images — this can take several minutes)…') diff --git a/scripts/setup/steps.test.ts b/scripts/setup/steps.test.ts index 2eb2d10183d..d28295c5ccc 100644 --- a/scripts/setup/steps.test.ts +++ b/scripts/setup/steps.test.ts @@ -2,11 +2,12 @@ import { describe, expect, it } from 'bun:test' import { EMAIL_CAPABILITY, inspectCapability, + KNOWLEDGE_EMBEDDINGS_CAPABILITY, requireCapability, STORAGE_CAPABILITY, validateCapabilityFieldInput, } from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { EMAIL_SETUP, STORAGE_SETUP } from './capability-config.ts' +import { EMAIL_SETUP, KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP } from './capability-config.ts' import { buildCapabilitySetupTransition, resolveCurrentCapabilitySetupOptionId, @@ -41,6 +42,15 @@ describe('setup provider reconciliation', () => { ).toBe('smtp') }) + it('allows setup when no knowledge embedding provider is configured', () => { + expect(resolveCurrentCapabilitySetupOptionId(KNOWLEDGE_EMBEDDINGS_SETUP, {})).toBeUndefined() + expect( + resolveCurrentCapabilitySetupOptionId(KNOWLEDGE_EMBEDDINGS_SETUP, { + OPENROUTER_API_KEY: 'openrouter-key', + }) + ).toBe('openrouter') + }) + it('uses canonical storage selection for setup defaults', () => { expect( resolveCurrentCapabilitySetupOptionId(STORAGE_SETUP, new Map([['AWS_REGION', 'us-east-1']])) @@ -79,6 +89,20 @@ describe('setup provider reconciliation', () => { expect(inspectCapability(EMAIL_CAPABILITY, reconciled).providerIds).toEqual(['resend', 'smtp']) }) + it('configures a knowledge embedding provider from an empty state', () => { + const result = buildCapabilitySetupTransition( + KNOWLEDGE_EMBEDDINGS_SETUP, + 'openrouter', + { OPENROUTER_API_KEY: 'openrouter-key' }, + {} + ) + const reconciled = applyResult({}, result) + + expect(inspectCapability(KNOWLEDGE_EMBEDDINGS_CAPABILITY, reconciled).providerIds).toEqual([ + 'openrouter', + ]) + }) + it('clears stale SMTP auth for an unauthenticated relay', () => { const result = buildCapabilitySetupTransition( EMAIL_SETUP, diff --git a/scripts/setup/steps.ts b/scripts/setup/steps.ts index caf3a49c226..91f05b4181c 100644 --- a/scripts/setup/steps.ts +++ b/scripts/setup/steps.ts @@ -1,3 +1,9 @@ +import { KNOWLEDGE_EMBEDDINGS_SETUP } from './capability-config.ts' +import { + type CapabilitySetupContext, + type EnvCapabilitySetupTransition, + promptOptionalCapabilitySetup, +} from './capability-setup.ts' import { browserKeyFlow } from './cli-auth.ts' import type { Detection } from './detect.ts' import { @@ -149,6 +155,19 @@ export async function promptLlmKeys( return values } +/** Configures knowledge embeddings while allowing the user to explicitly defer them. */ +export function promptKnowledgeEmbeddings( + currentValues: ReadonlyMap, + context: CapabilitySetupContext +): Promise { + return promptOptionalCapabilitySetup( + KNOWLEDGE_EMBEDDINGS_SETUP, + currentValues, + context, + 'knowledge-base indexing and semantic search will remain unavailable' + ) +} + const PROVIDER_CONSOLES: Record = { google: 'https://console.cloud.google.com/apis/credentials', github: 'https://github.com/settings/developers', diff --git a/scripts/setup/wizard.ts b/scripts/setup/wizard.ts index 76ad0628c44..b9de2835fd0 100644 --- a/scripts/setup/wizard.ts +++ b/scripts/setup/wizard.ts @@ -163,7 +163,7 @@ export async function runWizard(flags: WizardFlags): Promise { p.note( [ mode === 'k8s' ? `port-forward, then open ${APP_SIGNUP_URL}` : `open ${APP_SIGNUP_URL}`, - 'manage it: bun run sim start · stop · status · logs', + 'manage it: bun run sim start · stop · update · status · logs', 'check your setup: bun run sim doctor', mode === 'dev' && !startDevNow ? `start Sim: bun run ${devScript}` : null, `prefer a bare "sim"? ${theme.command('bun link')} once (needs ~/.bun/bin on PATH)`,