From c85d0685b4d7cb9b8421b3db8786c94dcb662cee Mon Sep 17 00:00:00 2001 From: waleed Date: Mon, 15 Jun 2026 16:35:55 -0700 Subject: [PATCH 01/10] refactor(connectors): split client metadata from server runtime + cover node:net in client bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser build broke with `Cannot find module 'node:net'`. Server-only SSRF code in `input-validation.server.ts` (`dns/promises`, and since PR #5060 `undici` → `node:net`/`node:tls`) is statically reachable from the client bundle via the tool/connector registries, which the workflow editor imports for metadata. Node networking builtins have no browser shim, so Turbopack cannot compile them for the client. Two changes: 1. Split each connector's client-safe declarative metadata into a sibling `meta.ts` (`ConnectorMeta`), mirroring the `XBlockMeta` / `BLOCK_META_REGISTRY` pattern. `connectors/registry.ts` is now the client-safe `CONNECTOR_META_REGISTRY` (+ `getConnectorMeta` / `getAllConnectorMeta`); the full registry with runtime fns moves to `connectors/registry.server.ts`. Client components consume the meta registry; the sync engine and knowledge API routes consume the server registry. This removes connectors from the client's server-only graph. Connector metadata is byte-for-byte identical before/after; runtime fns are untouched. 2. Extend the existing #4899 `turbopack.resolveAlias` browser stub — which already mapped `dns`/`dns/promises` to an empty module for the browser — to also cover `net`/`tls` (+ `node:` variants), since `undici` now pulls those in. The remaining tool/provider definitions still reach `input-validation.server` server-side; the browser-only stub keeps those Node builtins out of the client bundle while the real modules stay on the server, so SSRF validation and IP pinning are unaffected. Connector authoring/validation skills updated to teach the meta.ts split. --- .agents/skills/add-connector/SKILL.md | 81 +++++-- .agents/skills/validate-connector/SKILL.md | 27 ++- .claude/commands/add-connector.md | 81 +++++-- .claude/commands/validate-connector.md | 27 ++- .cursor/commands/add-connector.md | 81 +++++-- .cursor/commands/validate-connector.md | 27 ++- .../connectors/[connectorId]/route.test.ts | 2 +- .../[id]/connectors/[connectorId]/route.ts | 2 +- .../api/knowledge/[id]/connectors/route.ts | 2 +- .../knowledge/[id]/[documentId]/document.tsx | 4 +- .../[workspaceId]/knowledge/[id]/base.tsx | 8 +- .../add-connector-modal.tsx | 10 +- .../connector-config-fields.tsx | 4 +- .../connectors-section/connectors-section.tsx | 4 +- .../edit-connector-modal.tsx | 8 +- .../[id]/hooks/use-connector-config-fields.ts | 4 +- .../components/base-card/base-card.tsx | 4 +- .../[workspaceId]/knowledge/knowledge.tsx | 9 +- apps/sim/connectors/airtable/airtable.ts | 79 +----- apps/sim/connectors/airtable/meta.ts | 81 +++++++ apps/sim/connectors/asana/asana.ts | 54 +---- apps/sim/connectors/asana/meta.ts | 56 +++++ apps/sim/connectors/ashby/ashby.ts | 47 +--- apps/sim/connectors/ashby/meta.ts | 49 ++++ .../connectors/azure-devops/azure-devops.ts | 175 +------------- apps/sim/connectors/azure-devops/meta.ts | 177 ++++++++++++++ apps/sim/connectors/confluence/confluence.ts | 85 +------ apps/sim/connectors/confluence/meta.ts | 87 +++++++ apps/sim/connectors/discord/discord.ts | 38 +-- apps/sim/connectors/discord/meta.ts | 42 ++++ apps/sim/connectors/docusign/docusign.ts | 66 +---- apps/sim/connectors/docusign/meta.ts | 68 ++++++ apps/sim/connectors/dropbox/dropbox.ts | 38 +-- apps/sim/connectors/dropbox/meta.ts | 40 ++++ apps/sim/connectors/evernote/evernote.ts | 32 +-- apps/sim/connectors/evernote/meta.ts | 34 +++ apps/sim/connectors/fathom/fathom.ts | 77 +----- apps/sim/connectors/fathom/meta.ts | 79 ++++++ apps/sim/connectors/fireflies/fireflies.ts | 39 +-- apps/sim/connectors/fireflies/meta.ts | 41 ++++ apps/sim/connectors/github/github.ts | 60 +---- apps/sim/connectors/github/meta.ts | 62 +++++ apps/sim/connectors/gitlab/gitlab.ts | 138 +---------- apps/sim/connectors/gitlab/meta.ts | 140 +++++++++++ apps/sim/connectors/gmail/gmail.ts | 97 +------- apps/sim/connectors/gmail/meta.ts | 100 ++++++++ apps/sim/connectors/gong/gong.ts | 70 +----- apps/sim/connectors/gong/meta.ts | 72 ++++++ .../google-calendar/google-calendar.ts | 78 +----- apps/sim/connectors/google-calendar/meta.ts | 81 +++++++ .../sim/connectors/google-docs/google-docs.ts | 36 +-- apps/sim/connectors/google-docs/meta.ts | 38 +++ .../connectors/google-drive/google-drive.ts | 51 +--- apps/sim/connectors/google-drive/meta.ts | 53 +++++ .../connectors/google-forms/google-forms.ts | 69 +----- apps/sim/connectors/google-forms/meta.ts | 71 ++++++ .../connectors/google-sheets/google-sheets.ts | 42 +--- apps/sim/connectors/google-sheets/meta.ts | 44 ++++ apps/sim/connectors/grain/grain.ts | 88 +------ apps/sim/connectors/grain/meta.ts | 90 +++++++ apps/sim/connectors/granola/granola.ts | 67 +----- apps/sim/connectors/granola/meta.ts | 69 ++++++ apps/sim/connectors/greenhouse/greenhouse.ts | 70 +----- apps/sim/connectors/greenhouse/meta.ts | 72 ++++++ apps/sim/connectors/hubspot/hubspot.ts | 48 +--- apps/sim/connectors/hubspot/meta.ts | 50 ++++ apps/sim/connectors/incidentio/incidentio.ts | 74 +----- apps/sim/connectors/incidentio/meta.ts | 76 ++++++ apps/sim/connectors/intercom/intercom.ts | 71 +----- apps/sim/connectors/intercom/meta.ts | 74 ++++++ apps/sim/connectors/jira/jira.ts | 65 +---- apps/sim/connectors/jira/meta.ts | 67 ++++++ apps/sim/connectors/jsm/jsm.ts | 141 +---------- apps/sim/connectors/jsm/meta.ts | 143 +++++++++++ apps/sim/connectors/linear/linear.ts | 78 +----- apps/sim/connectors/linear/meta.ts | 80 +++++++ apps/sim/connectors/microsoft-teams/meta.ts | 77 ++++++ .../microsoft-teams/microsoft-teams.ts | 77 +----- apps/sim/connectors/monday/meta.ts | 45 ++++ apps/sim/connectors/monday/monday.ts | 43 +--- apps/sim/connectors/notion/meta.ts | 74 ++++++ apps/sim/connectors/notion/notion.ts | 72 +----- apps/sim/connectors/obsidian/meta.ts | 42 ++++ apps/sim/connectors/obsidian/obsidian.ts | 40 +--- apps/sim/connectors/onedrive/meta.ts | 29 +++ apps/sim/connectors/onedrive/onedrive.ts | 27 +-- apps/sim/connectors/outlook/meta.ts | 92 +++++++ apps/sim/connectors/outlook/outlook.ts | 89 +------ apps/sim/connectors/reddit/meta.ts | 72 ++++++ apps/sim/connectors/reddit/reddit.ts | 69 +----- apps/sim/connectors/registry.server.ts | 111 +++++++++ apps/sim/connectors/registry.ts | 225 ++++++++++-------- apps/sim/connectors/rootly/meta.ts | 88 +++++++ apps/sim/connectors/rootly/rootly.ts | 86 +------ apps/sim/connectors/s3/meta.ts | 83 +++++++ apps/sim/connectors/s3/s3.ts | 81 +------ apps/sim/connectors/salesforce/meta.ts | 41 ++++ apps/sim/connectors/salesforce/salesforce.ts | 39 +-- apps/sim/connectors/sentry/meta.ts | 108 +++++++++ apps/sim/connectors/sentry/sentry.ts | 106 +-------- apps/sim/connectors/servicenow/meta.ts | 119 +++++++++ apps/sim/connectors/servicenow/servicenow.ts | 116 +-------- apps/sim/connectors/sharepoint/meta.ts | 44 ++++ apps/sim/connectors/sharepoint/sharepoint.ts | 42 +--- apps/sim/connectors/slack/meta.ts | 63 +++++ apps/sim/connectors/slack/slack.ts | 60 +---- apps/sim/connectors/typeform/meta.ts | 88 +++++++ apps/sim/connectors/typeform/typeform.ts | 86 +------ apps/sim/connectors/types.ts | 52 ++-- .../{mapTags.test.ts => utils.test.ts} | 0 apps/sim/connectors/webflow/meta.ts | 55 +++++ apps/sim/connectors/webflow/webflow.ts | 53 +---- apps/sim/connectors/wordpress/meta.ts | 54 +++++ apps/sim/connectors/wordpress/wordpress.ts | 51 +--- apps/sim/connectors/x/meta.ts | 103 ++++++++ apps/sim/connectors/x/x.ts | 100 +------- apps/sim/connectors/youtube/meta.ts | 72 ++++++ apps/sim/connectors/youtube/youtube.ts | 70 +----- apps/sim/connectors/zendesk/meta.ts | 91 +++++++ apps/sim/connectors/zendesk/zendesk.ts | 88 +------ apps/sim/connectors/zoom/meta.ts | 52 ++++ apps/sim/connectors/zoom/zoom.ts | 50 +--- apps/sim/lib/copilot/vfs/workspace-vfs.ts | 2 +- .../security/empty-node-fallback.browser.ts | 17 +- .../knowledge/connectors/sync-engine.test.ts | 2 +- .../lib/knowledge/connectors/sync-engine.ts | 2 +- apps/sim/next.config.ts | 10 +- 127 files changed, 4305 insertions(+), 3647 deletions(-) create mode 100644 apps/sim/connectors/airtable/meta.ts create mode 100644 apps/sim/connectors/asana/meta.ts create mode 100644 apps/sim/connectors/ashby/meta.ts create mode 100644 apps/sim/connectors/azure-devops/meta.ts create mode 100644 apps/sim/connectors/confluence/meta.ts create mode 100644 apps/sim/connectors/discord/meta.ts create mode 100644 apps/sim/connectors/docusign/meta.ts create mode 100644 apps/sim/connectors/dropbox/meta.ts create mode 100644 apps/sim/connectors/evernote/meta.ts create mode 100644 apps/sim/connectors/fathom/meta.ts create mode 100644 apps/sim/connectors/fireflies/meta.ts create mode 100644 apps/sim/connectors/github/meta.ts create mode 100644 apps/sim/connectors/gitlab/meta.ts create mode 100644 apps/sim/connectors/gmail/meta.ts create mode 100644 apps/sim/connectors/gong/meta.ts create mode 100644 apps/sim/connectors/google-calendar/meta.ts create mode 100644 apps/sim/connectors/google-docs/meta.ts create mode 100644 apps/sim/connectors/google-drive/meta.ts create mode 100644 apps/sim/connectors/google-forms/meta.ts create mode 100644 apps/sim/connectors/google-sheets/meta.ts create mode 100644 apps/sim/connectors/grain/meta.ts create mode 100644 apps/sim/connectors/granola/meta.ts create mode 100644 apps/sim/connectors/greenhouse/meta.ts create mode 100644 apps/sim/connectors/hubspot/meta.ts create mode 100644 apps/sim/connectors/incidentio/meta.ts create mode 100644 apps/sim/connectors/intercom/meta.ts create mode 100644 apps/sim/connectors/jira/meta.ts create mode 100644 apps/sim/connectors/jsm/meta.ts create mode 100644 apps/sim/connectors/linear/meta.ts create mode 100644 apps/sim/connectors/microsoft-teams/meta.ts create mode 100644 apps/sim/connectors/monday/meta.ts create mode 100644 apps/sim/connectors/notion/meta.ts create mode 100644 apps/sim/connectors/obsidian/meta.ts create mode 100644 apps/sim/connectors/onedrive/meta.ts create mode 100644 apps/sim/connectors/outlook/meta.ts create mode 100644 apps/sim/connectors/reddit/meta.ts create mode 100644 apps/sim/connectors/registry.server.ts create mode 100644 apps/sim/connectors/rootly/meta.ts create mode 100644 apps/sim/connectors/s3/meta.ts create mode 100644 apps/sim/connectors/salesforce/meta.ts create mode 100644 apps/sim/connectors/sentry/meta.ts create mode 100644 apps/sim/connectors/servicenow/meta.ts create mode 100644 apps/sim/connectors/sharepoint/meta.ts create mode 100644 apps/sim/connectors/slack/meta.ts create mode 100644 apps/sim/connectors/typeform/meta.ts rename apps/sim/connectors/{mapTags.test.ts => utils.test.ts} (100%) create mode 100644 apps/sim/connectors/webflow/meta.ts create mode 100644 apps/sim/connectors/wordpress/meta.ts create mode 100644 apps/sim/connectors/x/meta.ts create mode 100644 apps/sim/connectors/youtube/meta.ts create mode 100644 apps/sim/connectors/zendesk/meta.ts create mode 100644 apps/sim/connectors/zoom/meta.ts diff --git a/.agents/skills/add-connector/SKILL.md b/.agents/skills/add-connector/SKILL.md index 0fccb938b11..2958623e156 100644 --- a/.agents/skills/add-connector/SKILL.md +++ b/.agents/skills/add-connector/SKILL.md @@ -12,8 +12,8 @@ You are an expert at adding knowledge base connectors to Sim. A connector syncs When the user asks you to create a connector: 1. Use Context7 or WebFetch to read the service's API documentation 2. Determine the auth mode: **OAuth** (if Sim already has an OAuth provider for the service) or **API key** (if the service uses API key / Bearer token auth) -3. Create the connector directory and config -4. Register it in the connector registry +3. Create the connector directory: a client-safe `meta.ts` (declarative metadata) plus the runtime module that spreads it +4. Register it in BOTH the server registry and the client-safe meta registry ## Hard Rule: No Guessed Response Or Document Schemas @@ -32,13 +32,19 @@ If the source schema is unknown, do one of these instead: ## Directory Structure +Each connector is split into a client-safe metadata file and a server-only runtime file. This mirrors the `XBlockMeta` / `BLOCK_META_REGISTRY` split in `apps/sim/blocks` — client components (the knowledge UI) only need the metadata (icon, name, auth, config fields), so the runtime functions (which pull server-only helpers like `input-validation.server` → `undici` → `node:net`) must stay out of the client bundle. + Create files in `apps/sim/connectors/{service}/`: ``` connectors/{service}/ -├── index.ts # Barrel export -└── {service}.ts # ConnectorConfig definition +├── index.ts # Barrel export (re-exports the runtime connector) +├── meta.ts # ConnectorMeta — client-safe declarative metadata +└── {service}.ts # ConnectorConfig — spreads the meta + adds runtime functions ``` +- `meta.ts` exports `{service}ConnectorMeta: ConnectorMeta`. It imports ONLY the icon from `@/components/icons`, `import type { ConnectorMeta } from '@/connectors/types'`, and any pure-data constants. It must NEVER import server/runtime code. +- `{service}.ts` exports `{service}Connector: ConnectorConfig`. It imports the meta via `import { {service}ConnectorMeta } from '@/connectors/{service}/meta'`, spreads it as the first property, and holds the runtime functions (which may import server-only helpers like `@/lib/knowledge/documents/utils`). + ## Authentication Connectors use a discriminated union for auth config (`ConnectorAuthConfig` in `connectors/types.ts`): @@ -55,19 +61,17 @@ For services with existing OAuth providers in `apps/sim/lib/oauth/types.ts`. The ### API key mode For services that use API key / Bearer token auth. The modal shows a password input with the configured `label` and `placeholder`. The API key is encrypted at rest using AES-256-GCM and stored in a dedicated `encryptedApiKey` column on the connector record. The sync engine decrypts it automatically — connectors receive the raw access token in `listDocuments`, `getDocument`, and `validateConfig`. -## ConnectorConfig Structure +## Connector Structure (meta.ts + runtime) + +The declarative metadata lives in `meta.ts` (`ConnectorMeta`). The runtime functions live in `{service}.ts` (`ConnectorConfig`), which spreads the meta as its first property. -### OAuth connector example +### `meta.ts` — client-safe metadata ```typescript -import { createLogger } from '@sim/logger' import { {Service}Icon } from '@/components/icons' -import { fetchWithRetry } from '@/lib/knowledge/documents/utils' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' - -const logger = createLogger('{Service}Connector') +import type { ConnectorMeta } from '@/connectors/types' -export const {service}Connector: ConnectorConfig = { +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -85,6 +89,26 @@ export const {service}Connector: ConnectorConfig = { // Supports 'short-input' and 'dropdown' types ], + // Optional: tag definitions are metadata too — declare them here + // tagDefinitions: [ ... ], +} +``` + +Keep `meta.ts` free of any server/runtime import. Only the icon, the `ConnectorMeta` type, and pure-data constants belong here. + +### `{service}.ts` — runtime (OAuth example) + +```typescript +import { createLogger } from '@sim/logger' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' + +const logger = createLogger('{Service}Connector') + +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, + listDocuments: async (accessToken, sourceConfig, cursor) => { // Return metadata stubs with contentDeferred: true (if per-doc content fetch needed) // Or full documents with content (if list API returns content inline) @@ -111,8 +135,11 @@ Only map fields in `listDocuments`, `getDocument`, `validateConfig`, and `mapTag ### API key connector example +The split is identical — `auth` lives in `meta.ts`, runtime functions in `{service}.ts`. + ```typescript -export const {service}Connector: ConnectorConfig = { +// meta.ts +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -126,6 +153,11 @@ export const {service}Connector: ConnectorConfig = { }, configFields: [ /* ... */ ], +} + +// {service}.ts +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, listDocuments: async (accessToken, sourceConfig, cursor) => { /* ... */ }, getDocument: async (accessToken, sourceConfig, externalId) => { /* ... */ }, validateConfig: async (accessToken, sourceConfig) => { /* ... */ }, @@ -499,7 +531,9 @@ If the service already has an icon in `apps/sim/components/icons.tsx` (from a to ## Registering -Add one line to `apps/sim/connectors/registry.ts`: +Register in BOTH registries, keeping the same alphabetical-by-id ordering in each. + +1. **Server registry** — `apps/sim/connectors/registry.server.ts` (server-only full registry; holds full connectors with runtime functions, imported by the sync engine and knowledge API routes): ```typescript import { {service}Connector } from '@/connectors/{service}' @@ -510,6 +544,19 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { } ``` +2. **Client-safe meta registry** — `apps/sim/connectors/registry.ts` (imports each connector's `meta.ts` only, so client components can use it without pulling server-only code; the metadata counterpart to `BLOCK_META_REGISTRY`): + +```typescript +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' + +export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { + // ... existing connector metas ... + {service}: {service}ConnectorMeta, +} +``` + +`registry.ts` exports `CONNECTOR_META_REGISTRY: ConnectorMetaRegistry` plus the helpers `getConnectorMeta(id)` and `getAllConnectorMeta()`, importing each `@/connectors/{service}/meta` directly — never the runtime module. `registry.server.ts` exports `CONNECTOR_REGISTRY: ConnectorRegistry`. + ## Reference Implementations - **OAuth + contentDeferred**: `apps/sim/connectors/google-drive/google-drive.ts` — file download with metadata-based hash, `orderBy` for deterministic pagination @@ -520,7 +567,8 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { ## Checklist -- [ ] Created `connectors/{service}/{service}.ts` with full ConnectorConfig +- [ ] Created `connectors/{service}/meta.ts` with `{service}ConnectorMeta: ConnectorMeta` (icon, name, auth, configFields, tagDefinitions) — no server/runtime imports +- [ ] Created `connectors/{service}/{service}.ts` with `{service}Connector: ConnectorConfig` spreading the meta + runtime functions - [ ] Created `connectors/{service}/index.ts` barrel export - [ ] **Auth configured correctly:** - OAuth: `auth.provider` matches an existing `OAuthService` in `lib/oauth/types.ts` @@ -542,4 +590,5 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { - [ ] All external API calls use `fetchWithRetry` (not raw `fetch`) - [ ] All optional config fields validated in `validateConfig` - [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG) -- [ ] Registered in `connectors/registry.ts` +- [ ] Registered the full connector in `connectors/registry.server.ts` +- [ ] Registered the meta in `connectors/registry.ts` (same alphabetical-by-id ordering as registry.server.ts) diff --git a/.agents/skills/validate-connector/SKILL.md b/.agents/skills/validate-connector/SKILL.md index 1a0024ace07..31d4f4ac4ef 100644 --- a/.agents/skills/validate-connector/SKILL.md +++ b/.agents/skills/validate-connector/SKILL.md @@ -21,10 +21,12 @@ When the user asks you to validate a connector: Read **every** file for the connector — do not skip any: ``` -apps/sim/connectors/{service}/{service}.ts # Connector implementation +apps/sim/connectors/{service}/meta.ts # ConnectorMeta — client-safe metadata (icon, name, auth, configFields, tagDefinitions) +apps/sim/connectors/{service}/{service}.ts # Connector implementation — spreads the meta + runtime functions apps/sim/connectors/{service}/index.ts # Barrel export -apps/sim/connectors/registry.ts # Connector registry entry -apps/sim/connectors/types.ts # ConnectorConfig interface, ExternalDocument, etc. +apps/sim/connectors/registry.server.ts # Server-only full registry entry (CONNECTOR_REGISTRY; full connector) +apps/sim/connectors/registry.ts # Client-safe meta registry entry (CONNECTOR_META_REGISTRY) +apps/sim/connectors/types.ts # ConnectorMeta / ConnectorConfig interfaces, ExternalDocument, etc. apps/sim/connectors/utils.ts # Shared utilities (computeContentHash, htmlToPlainText, etc.) apps/sim/lib/oauth/oauth.ts # OAUTH_PROVIDERS — single source of truth for scopes apps/sim/lib/oauth/utils.ts # getCanonicalScopesForProvider, getScopesForService, SCOPE_DESCRIPTIONS @@ -262,10 +264,18 @@ Connectors where the list API already returns content inline (e.g., Slack messag - [ ] Logs sync progress at `info` level - [ ] Logs errors at `warn` or `error` level with context +### Meta / Runtime Split +- [ ] `connectors/{service}/meta.ts` exports `{service}ConnectorMeta: ConnectorMeta` (id, name, description, version, icon, auth, configFields, and any `tagDefinitions` / `supportsIncrementalSync`) +- [ ] `meta.ts` imports ONLY the icon from `@/components/icons`, `ConnectorMeta` (type-only), and pure-data constants — NO server/runtime imports (`@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`, etc.); any such import in `meta.ts` is **critical** (breaks the client bundle) +- [ ] `connectors/{service}/{service}.ts` spreads `...{service}ConnectorMeta` as the first property and adds the runtime functions (`listDocuments`, `getDocument`, `validateConfig`, `mapTags?`) +- [ ] Metadata fields (id, name, auth, configFields, etc.) live ONLY in `meta.ts`, not duplicated in `{service}.ts` + ### Registry - [ ] Connector is exported from `connectors/{service}/index.ts` -- [ ] Connector is registered in `connectors/registry.ts` -- [ ] Registry key matches the connector's `id` field +- [ ] Full connector is registered in `connectors/registry.server.ts` (server-only registry, `CONNECTOR_REGISTRY`) +- [ ] Meta is registered in `connectors/registry.ts` (client-safe registry, `CONNECTOR_META_REGISTRY`), importing `@/connectors/{service}/meta` +- [ ] Both registries use the same key and it matches the connector's `id` field +- [ ] Both registries keep the same alphabetical-by-id ordering ## Step 11: Report and Fix @@ -284,6 +294,8 @@ Group findings by severity: - Query/filter injection: user-controlled values interpolated into OData `$filter`, SOQL, or query strings without escaping - Per-document content download in `listDocuments` without `contentDeferred: true` — causes sync timeouts for large document sets - `contentHash` mismatch between `listDocuments` stub and `getDocument` return — causes unnecessary re-processing every sync +- Server/runtime import in `meta.ts` (e.g. `@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`) — pulls server-only code into the client bundle and breaks the build +- Connector missing from `connectors/registry.ts` (the client-safe meta registry) — or its entry there imports the runtime module instead of `meta.ts` — the knowledge UI can't render it **Warning** (incorrect behavior, data quality issues, or convention violations): - HTML content not stripped via `htmlToPlainText` @@ -323,7 +335,7 @@ After fixing, confirm: ## Checklist Summary -- [ ] Read connector implementation, types, utils, registry, and OAuth config +- [ ] Read connector meta.ts, implementation, types, utils, both registries, and OAuth config - [ ] Pulled and read official API documentation for the service - [ ] Validated every API endpoint URL, method, headers, and body against API docs - [ ] Validated input sanitization: no query/filter injection, URL fields normalized @@ -342,7 +354,8 @@ After fixing, confirm: - [ ] Validated API efficiency: field selection used, no redundant calls, sequential fetches batched - [ ] Validated error handling: graceful failures, no unhandled rejections - [ ] Validated logging: createLogger, no console.log -- [ ] Validated registry: correct export, correct key +- [ ] Validated meta/runtime split: `meta.ts` holds metadata with no server/runtime imports, `{service}.ts` spreads the meta + adds runtime functions +- [ ] Validated registry: exported from index.ts, full connector in `registry.server.ts`, meta in `registry.ts`, matching keys and alphabetical-by-id ordering in both - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run lint` after fixes diff --git a/.claude/commands/add-connector.md b/.claude/commands/add-connector.md index 8d667ebb7a2..9b7a5650d09 100644 --- a/.claude/commands/add-connector.md +++ b/.claude/commands/add-connector.md @@ -12,18 +12,24 @@ You are an expert at adding knowledge base connectors to Sim. A connector syncs When the user asks you to create a connector: 1. Use Context7 or WebFetch to read the service's API documentation 2. Determine the auth mode: **OAuth** (if Sim already has an OAuth provider for the service) or **API key** (if the service uses API key / Bearer token auth) -3. Create the connector directory and config -4. Register it in the connector registry +3. Create the connector directory: a client-safe `meta.ts` (declarative metadata) plus the runtime module that spreads it +4. Register it in BOTH the server registry and the client-safe meta registry ## Directory Structure +Each connector is split into a client-safe metadata file and a server-only runtime file. This mirrors the `XBlockMeta` / `BLOCK_META_REGISTRY` split in `apps/sim/blocks` — client components (the knowledge UI) only need the metadata (icon, name, auth, config fields), so the runtime functions (which pull server-only helpers like `input-validation.server` → `undici` → `node:net`) must stay out of the client bundle. + Create files in `apps/sim/connectors/{service}/`: ``` connectors/{service}/ -├── index.ts # Barrel export -└── {service}.ts # ConnectorConfig definition +├── index.ts # Barrel export (re-exports the runtime connector) +├── meta.ts # ConnectorMeta — client-safe declarative metadata +└── {service}.ts # ConnectorConfig — spreads the meta + adds runtime functions ``` +- `meta.ts` exports `{service}ConnectorMeta: ConnectorMeta`. It imports ONLY the icon from `@/components/icons`, `import type { ConnectorMeta } from '@/connectors/types'`, and any pure-data constants. It must NEVER import server/runtime code. +- `{service}.ts` exports `{service}Connector: ConnectorConfig`. It imports the meta via `import { {service}ConnectorMeta } from '@/connectors/{service}/meta'`, spreads it as the first property, and holds the runtime functions (which may import server-only helpers like `@/lib/knowledge/documents/utils`). + ## Authentication Connectors use a discriminated union for auth config (`ConnectorAuthConfig` in `connectors/types.ts`): @@ -40,19 +46,17 @@ For services with existing OAuth providers in `apps/sim/lib/oauth/types.ts`. The ### API key mode For services that use API key / Bearer token auth. The modal shows a password input with the configured `label` and `placeholder`. The API key is encrypted at rest using AES-256-GCM and stored in a dedicated `encryptedApiKey` column on the connector record. The sync engine decrypts it automatically — connectors receive the raw access token in `listDocuments`, `getDocument`, and `validateConfig`. -## ConnectorConfig Structure +## Connector Structure (meta.ts + runtime) + +The declarative metadata lives in `meta.ts` (`ConnectorMeta`). The runtime functions live in `{service}.ts` (`ConnectorConfig`), which spreads the meta as its first property. -### OAuth connector example +### `meta.ts` — client-safe metadata ```typescript -import { createLogger } from '@sim/logger' import { {Service}Icon } from '@/components/icons' -import { fetchWithRetry } from '@/lib/knowledge/documents/utils' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' - -const logger = createLogger('{Service}Connector') +import type { ConnectorMeta } from '@/connectors/types' -export const {service}Connector: ConnectorConfig = { +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -70,6 +74,26 @@ export const {service}Connector: ConnectorConfig = { // Supports 'short-input', 'dropdown', and 'selector' types — see ConfigField Types below ], + // Optional: tag definitions are metadata too — declare them here + // tagDefinitions: [ ... ], +} +``` + +Keep `meta.ts` free of any server/runtime import. Only the icon, the `ConnectorMeta` type, and pure-data constants belong here. + +### `{service}.ts` — runtime (OAuth example) + +```typescript +import { createLogger } from '@sim/logger' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' + +const logger = createLogger('{Service}Connector') + +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, + listDocuments: async (accessToken, sourceConfig, cursor) => { // Return metadata stubs with contentDeferred: true (if per-doc content fetch needed) // Or full documents with content (if list API returns content inline) @@ -94,8 +118,11 @@ export const {service}Connector: ConnectorConfig = { ### API key connector example +The split is identical — `auth` lives in `meta.ts`, runtime functions in `{service}.ts`. + ```typescript -export const {service}Connector: ConnectorConfig = { +// meta.ts +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -109,6 +136,11 @@ export const {service}Connector: ConnectorConfig = { }, configFields: [ /* ... */ ], +} + +// {service}.ts +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, listDocuments: async (accessToken, sourceConfig, cursor) => { /* ... */ }, getDocument: async (accessToken, sourceConfig, externalId) => { /* ... */ }, validateConfig: async (accessToken, sourceConfig) => { /* ... */ }, @@ -512,7 +544,9 @@ If the service already has an icon in `apps/sim/components/icons.tsx` (from a to ## Registering -Add one line to `apps/sim/connectors/registry.ts`: +Register in BOTH registries, keeping the same alphabetical-by-id ordering in each. + +1. **Server registry** — `apps/sim/connectors/registry.server.ts` (server-only full registry; holds full connectors with runtime functions, imported by the sync engine and knowledge API routes): ```typescript import { {service}Connector } from '@/connectors/{service}' @@ -523,6 +557,19 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { } ``` +2. **Client-safe meta registry** — `apps/sim/connectors/registry.ts` (imports each connector's `meta.ts` only, so client components can use it without pulling server-only code; the metadata counterpart to `BLOCK_META_REGISTRY`): + +```typescript +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' + +export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { + // ... existing connector metas ... + {service}: {service}ConnectorMeta, +} +``` + +`registry.ts` exports `CONNECTOR_META_REGISTRY: ConnectorMetaRegistry` plus the helpers `getConnectorMeta(id)` and `getAllConnectorMeta()`, importing each `@/connectors/{service}/meta` directly — never the runtime module. `registry.server.ts` exports `CONNECTOR_REGISTRY: ConnectorRegistry`. + ## Reference Implementations - **OAuth + contentDeferred**: `apps/sim/connectors/google-drive/google-drive.ts` — file download with metadata-based hash, `orderBy` for deterministic pagination @@ -533,7 +580,8 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { ## Checklist -- [ ] Created `connectors/{service}/{service}.ts` with full ConnectorConfig +- [ ] Created `connectors/{service}/meta.ts` with `{service}ConnectorMeta: ConnectorMeta` (icon, name, auth, configFields, tagDefinitions) — no server/runtime imports +- [ ] Created `connectors/{service}/{service}.ts` with `{service}Connector: ConnectorConfig` spreading the meta + runtime functions - [ ] Created `connectors/{service}/index.ts` barrel export - [ ] **Auth configured correctly:** - OAuth: `auth.provider` matches an existing `OAuthService` in `lib/oauth/types.ts` @@ -556,4 +604,5 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { - [ ] All external API calls use `fetchWithRetry` (not raw `fetch`) - [ ] All optional config fields validated in `validateConfig` - [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG) -- [ ] Registered in `connectors/registry.ts` +- [ ] Registered the full connector in `connectors/registry.server.ts` +- [ ] Registered the meta in `connectors/registry.ts` (same alphabetical-by-id ordering as registry.server.ts) diff --git a/.claude/commands/validate-connector.md b/.claude/commands/validate-connector.md index 6bf8dab4d53..eca711b4ddd 100644 --- a/.claude/commands/validate-connector.md +++ b/.claude/commands/validate-connector.md @@ -21,10 +21,12 @@ When the user asks you to validate a connector: Read **every** file for the connector — do not skip any: ``` -apps/sim/connectors/{service}/{service}.ts # Connector implementation +apps/sim/connectors/{service}/meta.ts # ConnectorMeta — client-safe metadata (icon, name, auth, configFields, tagDefinitions) +apps/sim/connectors/{service}/{service}.ts # Connector implementation — spreads the meta + runtime functions apps/sim/connectors/{service}/index.ts # Barrel export -apps/sim/connectors/registry.ts # Connector registry entry -apps/sim/connectors/types.ts # ConnectorConfig interface, ExternalDocument, etc. +apps/sim/connectors/registry.server.ts # Server-only full registry entry (CONNECTOR_REGISTRY; full connector) +apps/sim/connectors/registry.ts # Client-safe meta registry entry (CONNECTOR_META_REGISTRY) +apps/sim/connectors/types.ts # ConnectorMeta / ConnectorConfig interfaces, ExternalDocument, etc. apps/sim/connectors/utils.ts # Shared utilities (computeContentHash, htmlToPlainText, etc.) apps/sim/lib/oauth/oauth.ts # OAUTH_PROVIDERS — single source of truth for scopes apps/sim/lib/oauth/utils.ts # getCanonicalScopesForProvider, getScopesForService, SCOPE_DESCRIPTIONS @@ -240,10 +242,18 @@ This is the most common connector bug class — verify it explicitly against `sy - [ ] Logs sync progress at `info` level - [ ] Logs errors at `warn` or `error` level with context +### Meta / Runtime Split +- [ ] `connectors/{service}/meta.ts` exports `{service}ConnectorMeta: ConnectorMeta` (id, name, description, version, icon, auth, configFields, and any `tagDefinitions` / `supportsIncrementalSync`) +- [ ] `meta.ts` imports ONLY the icon from `@/components/icons`, `ConnectorMeta` (type-only), and pure-data constants — NO server/runtime imports (`@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`, etc.); any such import in `meta.ts` is **critical** (breaks the client bundle) +- [ ] `connectors/{service}/{service}.ts` spreads `...{service}ConnectorMeta` as the first property and adds the runtime functions (`listDocuments`, `getDocument`, `validateConfig`, `mapTags?`) +- [ ] Metadata fields (id, name, auth, configFields, etc.) live ONLY in `meta.ts`, not duplicated in `{service}.ts` + ### Registry - [ ] Connector is exported from `connectors/{service}/index.ts` -- [ ] Connector is registered in `connectors/registry.ts` -- [ ] Registry key matches the connector's `id` field +- [ ] Full connector is registered in `connectors/registry.server.ts` (server-only registry, `CONNECTOR_REGISTRY`) +- [ ] Meta is registered in `connectors/registry.ts` (client-safe registry, `CONNECTOR_META_REGISTRY`), importing `@/connectors/{service}/meta` +- [ ] Both registries use the same key and it matches the connector's `id` field +- [ ] Both registries keep the same alphabetical-by-id ordering ## Step 11: Report and Fix @@ -260,6 +270,8 @@ Group findings by severity: - Missing error handling that would crash the sync - `requiredScopes` not a subset of OAuth provider scopes - Query/filter injection: user-controlled values interpolated into OData `$filter`, SOQL, or query strings without escaping +- Server/runtime import in `meta.ts` (e.g. `@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`) — pulls server-only code into the client bundle and breaks the build +- Connector missing from `connectors/registry.ts` (the client-safe meta registry) — or its entry there imports the runtime module instead of `meta.ts` — the knowledge UI can't render it **Warning** (incorrect behavior, data quality issues, or convention violations): - HTML content not stripped via `htmlToPlainText` @@ -299,7 +311,7 @@ After fixing, confirm: ## Checklist Summary -- [ ] Read connector implementation, types, utils, registry, and OAuth config +- [ ] Read connector meta.ts, implementation, types, utils, both registries, and OAuth config - [ ] Pulled and read official API documentation for the service - [ ] Validated every API endpoint URL, method, headers, and body against API docs - [ ] Validated input sanitization: no query/filter injection, URL fields normalized @@ -317,7 +329,8 @@ After fixing, confirm: - [ ] Validated API efficiency: field selection used, no redundant calls, sequential fetches batched - [ ] Validated error handling: graceful failures, no unhandled rejections - [ ] Validated logging: createLogger, no console.log -- [ ] Validated registry: correct export, correct key +- [ ] Validated meta/runtime split: `meta.ts` holds metadata with no server/runtime imports, `{service}.ts` spreads the meta + adds runtime functions +- [ ] Validated registry: exported from index.ts, full connector in `registry.server.ts`, meta in `registry.ts`, matching keys and alphabetical-by-id ordering in both - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run lint` after fixes diff --git a/.cursor/commands/add-connector.md b/.cursor/commands/add-connector.md index 9d29f5aab62..63ed2cec4ae 100644 --- a/.cursor/commands/add-connector.md +++ b/.cursor/commands/add-connector.md @@ -7,18 +7,24 @@ You are an expert at adding knowledge base connectors to Sim. A connector syncs When the user asks you to create a connector: 1. Use Context7 or WebFetch to read the service's API documentation 2. Determine the auth mode: **OAuth** (if Sim already has an OAuth provider for the service) or **API key** (if the service uses API key / Bearer token auth) -3. Create the connector directory and config -4. Register it in the connector registry +3. Create the connector directory: a client-safe `meta.ts` (declarative metadata) plus the runtime module that spreads it +4. Register it in BOTH the server registry and the client-safe meta registry ## Directory Structure +Each connector is split into a client-safe metadata file and a server-only runtime file. This mirrors the `XBlockMeta` / `BLOCK_META_REGISTRY` split in `apps/sim/blocks` — client components (the knowledge UI) only need the metadata (icon, name, auth, config fields), so the runtime functions (which pull server-only helpers like `input-validation.server` → `undici` → `node:net`) must stay out of the client bundle. + Create files in `apps/sim/connectors/{service}/`: ``` connectors/{service}/ -├── index.ts # Barrel export -└── {service}.ts # ConnectorConfig definition +├── index.ts # Barrel export (re-exports the runtime connector) +├── meta.ts # ConnectorMeta — client-safe declarative metadata +└── {service}.ts # ConnectorConfig — spreads the meta + adds runtime functions ``` +- `meta.ts` exports `{service}ConnectorMeta: ConnectorMeta`. It imports ONLY the icon from `@/components/icons`, `import type { ConnectorMeta } from '@/connectors/types'`, and any pure-data constants. It must NEVER import server/runtime code. +- `{service}.ts` exports `{service}Connector: ConnectorConfig`. It imports the meta via `import { {service}ConnectorMeta } from '@/connectors/{service}/meta'`, spreads it as the first property, and holds the runtime functions (which may import server-only helpers like `@/lib/knowledge/documents/utils`). + ## Authentication Connectors use a discriminated union for auth config (`ConnectorAuthConfig` in `connectors/types.ts`): @@ -35,19 +41,17 @@ For services with existing OAuth providers in `apps/sim/lib/oauth/types.ts`. The ### API key mode For services that use API key / Bearer token auth. The modal shows a password input with the configured `label` and `placeholder`. The API key is encrypted at rest using AES-256-GCM and stored in a dedicated `encryptedApiKey` column on the connector record. The sync engine decrypts it automatically — connectors receive the raw access token in `listDocuments`, `getDocument`, and `validateConfig`. -## ConnectorConfig Structure +## Connector Structure (meta.ts + runtime) + +The declarative metadata lives in `meta.ts` (`ConnectorMeta`). The runtime functions live in `{service}.ts` (`ConnectorConfig`), which spreads the meta as its first property. -### OAuth connector example +### `meta.ts` — client-safe metadata ```typescript -import { createLogger } from '@sim/logger' import { {Service}Icon } from '@/components/icons' -import { fetchWithRetry } from '@/lib/knowledge/documents/utils' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' - -const logger = createLogger('{Service}Connector') +import type { ConnectorMeta } from '@/connectors/types' -export const {service}Connector: ConnectorConfig = { +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -65,6 +69,26 @@ export const {service}Connector: ConnectorConfig = { // Supports 'short-input' and 'dropdown' types ], + // Optional: tag definitions are metadata too — declare them here + // tagDefinitions: [ ... ], +} +``` + +Keep `meta.ts` free of any server/runtime import. Only the icon, the `ConnectorMeta` type, and pure-data constants belong here. + +### `{service}.ts` — runtime (OAuth example) + +```typescript +import { createLogger } from '@sim/logger' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' + +const logger = createLogger('{Service}Connector') + +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, + listDocuments: async (accessToken, sourceConfig, cursor) => { // Return metadata stubs with contentDeferred: true (if per-doc content fetch needed) // Or full documents with content (if list API returns content inline) @@ -89,8 +113,11 @@ export const {service}Connector: ConnectorConfig = { ### API key connector example +The split is identical — `auth` lives in `meta.ts`, runtime functions in `{service}.ts`. + ```typescript -export const {service}Connector: ConnectorConfig = { +// meta.ts +export const {service}ConnectorMeta: ConnectorMeta = { id: '{service}', name: '{Service}', description: 'Sync documents from {Service} into your knowledge base', @@ -104,6 +131,11 @@ export const {service}Connector: ConnectorConfig = { }, configFields: [ /* ... */ ], +} + +// {service}.ts +export const {service}Connector: ConnectorConfig = { + ...{service}ConnectorMeta, listDocuments: async (accessToken, sourceConfig, cursor) => { /* ... */ }, getDocument: async (accessToken, sourceConfig, externalId) => { /* ... */ }, validateConfig: async (accessToken, sourceConfig) => { /* ... */ }, @@ -477,7 +509,9 @@ If the service already has an icon in `apps/sim/components/icons.tsx` (from a to ## Registering -Add one line to `apps/sim/connectors/registry.ts`: +Register in BOTH registries, keeping the same alphabetical-by-id ordering in each. + +1. **Server registry** — `apps/sim/connectors/registry.server.ts` (server-only full registry; holds full connectors with runtime functions, imported by the sync engine and knowledge API routes): ```typescript import { {service}Connector } from '@/connectors/{service}' @@ -488,6 +522,19 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { } ``` +2. **Client-safe meta registry** — `apps/sim/connectors/registry.ts` (imports each connector's `meta.ts` only, so client components can use it without pulling server-only code; the metadata counterpart to `BLOCK_META_REGISTRY`): + +```typescript +import { {service}ConnectorMeta } from '@/connectors/{service}/meta' + +export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { + // ... existing connector metas ... + {service}: {service}ConnectorMeta, +} +``` + +`registry.ts` exports `CONNECTOR_META_REGISTRY: ConnectorMetaRegistry` plus the helpers `getConnectorMeta(id)` and `getAllConnectorMeta()`, importing each `@/connectors/{service}/meta` directly — never the runtime module. `registry.server.ts` exports `CONNECTOR_REGISTRY: ConnectorRegistry`. + ## Reference Implementations - **OAuth + contentDeferred**: `apps/sim/connectors/google-drive/google-drive.ts` — file download with metadata-based hash, `orderBy` for deterministic pagination @@ -498,7 +545,8 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { ## Checklist -- [ ] Created `connectors/{service}/{service}.ts` with full ConnectorConfig +- [ ] Created `connectors/{service}/meta.ts` with `{service}ConnectorMeta: ConnectorMeta` (icon, name, auth, configFields, tagDefinitions) — no server/runtime imports +- [ ] Created `connectors/{service}/{service}.ts` with `{service}Connector: ConnectorConfig` spreading the meta + runtime functions - [ ] Created `connectors/{service}/index.ts` barrel export - [ ] **Auth configured correctly:** - OAuth: `auth.provider` matches an existing `OAuthService` in `lib/oauth/types.ts` @@ -520,4 +568,5 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { - [ ] All external API calls use `fetchWithRetry` (not raw `fetch`) - [ ] All optional config fields validated in `validateConfig` - [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG) -- [ ] Registered in `connectors/registry.ts` +- [ ] Registered the full connector in `connectors/registry.server.ts` +- [ ] Registered the meta in `connectors/registry.ts` (same alphabetical-by-id ordering as registry.server.ts) diff --git a/.cursor/commands/validate-connector.md b/.cursor/commands/validate-connector.md index e1e319e2c0b..bfc4e2520a6 100644 --- a/.cursor/commands/validate-connector.md +++ b/.cursor/commands/validate-connector.md @@ -16,10 +16,12 @@ When the user asks you to validate a connector: Read **every** file for the connector — do not skip any: ``` -apps/sim/connectors/{service}/{service}.ts # Connector implementation +apps/sim/connectors/{service}/meta.ts # ConnectorMeta — client-safe metadata (icon, name, auth, configFields, tagDefinitions) +apps/sim/connectors/{service}/{service}.ts # Connector implementation — spreads the meta + runtime functions apps/sim/connectors/{service}/index.ts # Barrel export -apps/sim/connectors/registry.ts # Connector registry entry -apps/sim/connectors/types.ts # ConnectorConfig interface, ExternalDocument, etc. +apps/sim/connectors/registry.server.ts # Server-only full registry entry (CONNECTOR_REGISTRY; full connector) +apps/sim/connectors/registry.ts # Client-safe meta registry entry (CONNECTOR_META_REGISTRY) +apps/sim/connectors/types.ts # ConnectorMeta / ConnectorConfig interfaces, ExternalDocument, etc. apps/sim/connectors/utils.ts # Shared utilities (computeContentHash, htmlToPlainText, etc.) apps/sim/lib/oauth/oauth.ts # OAUTH_PROVIDERS — single source of truth for scopes apps/sim/lib/oauth/utils.ts # getCanonicalScopesForProvider, getScopesForService, SCOPE_DESCRIPTIONS @@ -228,10 +230,18 @@ For each API endpoint the connector calls: - [ ] Logs sync progress at `info` level - [ ] Logs errors at `warn` or `error` level with context +### Meta / Runtime Split +- [ ] `connectors/{service}/meta.ts` exports `{service}ConnectorMeta: ConnectorMeta` (id, name, description, version, icon, auth, configFields, and any `tagDefinitions` / `supportsIncrementalSync`) +- [ ] `meta.ts` imports ONLY the icon from `@/components/icons`, `ConnectorMeta` (type-only), and pure-data constants — NO server/runtime imports (`@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`, etc.); any such import in `meta.ts` is **critical** (breaks the client bundle) +- [ ] `connectors/{service}/{service}.ts` spreads `...{service}ConnectorMeta` as the first property and adds the runtime functions (`listDocuments`, `getDocument`, `validateConfig`, `mapTags?`) +- [ ] Metadata fields (id, name, auth, configFields, etc.) live ONLY in `meta.ts`, not duplicated in `{service}.ts` + ### Registry - [ ] Connector is exported from `connectors/{service}/index.ts` -- [ ] Connector is registered in `connectors/registry.ts` -- [ ] Registry key matches the connector's `id` field +- [ ] Full connector is registered in `connectors/registry.server.ts` (server-only registry, `CONNECTOR_REGISTRY`) +- [ ] Meta is registered in `connectors/registry.ts` (client-safe registry, `CONNECTOR_META_REGISTRY`), importing `@/connectors/{service}/meta` +- [ ] Both registries use the same key and it matches the connector's `id` field +- [ ] Both registries keep the same alphabetical-by-id ordering ## Step 11: Report and Fix @@ -248,6 +258,8 @@ Group findings by severity: - Missing error handling that would crash the sync - `requiredScopes` not a subset of OAuth provider scopes - Query/filter injection: user-controlled values interpolated into OData `$filter`, SOQL, or query strings without escaping +- Server/runtime import in `meta.ts` (e.g. `@/lib/knowledge/...`, `input-validation.server`, `fetchWithRetry`) — pulls server-only code into the client bundle and breaks the build +- Connector missing from `connectors/registry.ts` (the client-safe meta registry) — or its entry there imports the runtime module instead of `meta.ts` — the knowledge UI can't render it **Warning** (incorrect behavior, data quality issues, or convention violations): - HTML content not stripped via `htmlToPlainText` @@ -286,7 +298,7 @@ After fixing, confirm: ## Checklist Summary -- [ ] Read connector implementation, types, utils, registry, and OAuth config +- [ ] Read connector meta.ts, implementation, types, utils, both registries, and OAuth config - [ ] Pulled and read official API documentation for the service - [ ] Validated every API endpoint URL, method, headers, and body against API docs - [ ] Validated input sanitization: no query/filter injection, URL fields normalized @@ -304,7 +316,8 @@ After fixing, confirm: - [ ] Validated API efficiency: field selection used, no redundant calls, sequential fetches batched - [ ] Validated error handling: graceful failures, no unhandled rejections - [ ] Validated logging: createLogger, no console.log -- [ ] Validated registry: correct export, correct key +- [ ] Validated meta/runtime split: `meta.ts` holds metadata with no server/runtime imports, `{service}.ts` spreads the meta + adds runtime functions +- [ ] Validated registry: exported from index.ts, full connector in `registry.server.ts`, meta in `registry.ts`, matching keys and alphabetical-by-id ordering in both - [ ] Reported all issues grouped by severity - [ ] Fixed all critical and warning issues - [ ] Ran `bun run lint` after fixes diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts index 94e41e01ba9..f904b9b5d35 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts @@ -39,7 +39,7 @@ const mockCheckWriteAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseWrit vi.mock('@sim/db', () => ({ db: mockDbChain })) vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) -vi.mock('@/connectors/registry', () => ({ +vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { jira: { validateConfig: mockValidateConfig }, }, diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts index d2e4ff37804..2e5bb4ee6cf 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts @@ -22,7 +22,7 @@ import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -import { CONNECTOR_REGISTRY } from '@/connectors/registry' +import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' const logger = createLogger('KnowledgeConnectorByIdAPI') diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index 3a9d6d61785..8cc3cc59621 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -18,7 +18,7 @@ import { createTagDefinition } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { getCredential } from '@/app/api/auth/oauth/utils' import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -import { CONNECTOR_REGISTRY } from '@/connectors/registry' +import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' const logger = createLogger('KnowledgeConnectorsAPI') diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index f62abbd9227..2266d160c7a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -35,7 +35,7 @@ import { ActionBar } from '@/app/workspace/[workspaceId]/knowledge/[id]/componen import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' -import { CONNECTOR_REGISTRY } from '@/connectors/registry' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { useDocument, useDocumentChunks, useKnowledgeBase } from '@/hooks/kb/use-knowledge' import { useBulkChunkOperation, @@ -278,7 +278,7 @@ export function Document({ const knowledgeBaseCrumbLabel = knowledgeBase?.name || knowledgeBaseName || '…' const documentCrumbLabel = documentData?.filename || documentName || '…' const ConnectorIcon = documentData?.connectorType - ? CONNECTOR_REGISTRY[documentData.connectorType]?.icon + ? CONNECTOR_META_REGISTRY[documentData.connectorType]?.icon : null const DocumentIcon = ConnectorIcon || getDocumentIcon(documentData?.mimeType ?? '', effectiveDocumentName) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 095979ed277..51572bc459e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -61,7 +61,7 @@ import { import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' -import { CONNECTOR_REGISTRY } from '@/connectors/registry' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { useKnowledgeBase, useKnowledgeBaseDocuments, @@ -926,7 +926,7 @@ export function KnowledgeBase({ connectors.length > 0 ? ( <> {connectors.map((connector) => { - const def = CONNECTOR_REGISTRY[connector.connectorType] + const def = CONNECTOR_META_REGISTRY[connector.connectorType] const ConnectorIcon = def?.icon return (