diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index e40d9af2efd..5d9b3efae5c 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -2769,14 +2769,6 @@ export function ExtendIcon(props: SVGProps) { ) } -export function EvernoteIcon(props: SVGProps) { - return ( - - - - ) -} - export function ElevenLabsIcon(props: SVGProps) { return ( = { enrich: EnrichSoIcon, enrichment: EnrichmentIcon, enrow: EnrowIcon, - evernote: EvernoteIcon, exa: ExaAIIcon, extend: ExtendIcon, extend_v2: ExtendIcon, diff --git a/apps/docs/content/docs/en/integrations/evernote.mdx b/apps/docs/content/docs/en/integrations/evernote.mdx deleted file mode 100644 index 9ad90dd9b7e..00000000000 --- a/apps/docs/content/docs/en/integrations/evernote.mdx +++ /dev/null @@ -1,282 +0,0 @@ ---- -title: Evernote -description: Manage notes, notebooks, and tags in Evernote ---- - -import { BlockInfoCard } from "@/components/ui/block-info-card" - - - -{/* MANUAL-CONTENT-START:intro */} -[Evernote](https://evernote.com/) is a note-taking and organization platform that helps individuals and teams capture ideas, manage projects, and store information across devices. With notebooks, tags, and powerful search, Evernote serves as a central hub for knowledge management. - -With the Sim Evernote integration, you can: - -- **Create and update notes**: Programmatically create new notes with content and tags, or update existing notes in any notebook. -- **Search and retrieve notes**: Use Evernote's search grammar to find notes by keyword, tag, notebook, or other criteria, and retrieve full note content. -- **Organize with notebooks and tags**: Create notebooks and tags, list existing ones, and move or copy notes between notebooks. -- **Delete and manage notes**: Move notes to trash or copy them to different notebooks as part of automated workflows. - -**How it works in Sim:** -Add an Evernote block to your workflow and select an operation (e.g., create note, search notes, list notebooks). Provide your Evernote developer token and any required parameters. The block calls the Evernote API and returns structured data you can pass to downstream blocks — for example, searching for meeting notes and sending summaries to Slack, or creating notes from AI-generated content. -{/* MANUAL-CONTENT-END */} - - -## Usage Instructions - -Integrate with Evernote to manage notes, notebooks, and tags. Create, read, update, copy, search, and delete notes. Create and list notebooks and tags. - - - -## Actions - -### Evernote Copy Note - -Copy a note to another notebook in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `noteGuid` | string | Yes | GUID of the note to copy | -| `toNotebookGuid` | string | Yes | GUID of the destination notebook | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `note` | object | The copied note metadata | -| ↳ `guid` | string | New note GUID | -| ↳ `title` | string | Note title | -| ↳ `notebookGuid` | string | GUID of the destination notebook | -| ↳ `created` | number | Creation timestamp in milliseconds | -| ↳ `updated` | number | Last updated timestamp in milliseconds | - -### Evernote Create Note - -Create a new note in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `title` | string | Yes | Title of the note | -| `content` | string | Yes | Content of the note \(plain text or ENML\) | -| `notebookGuid` | string | No | GUID of the notebook to create the note in \(defaults to default notebook\) | -| `tagNames` | string | No | Comma-separated list of tag names to apply | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `note` | object | The created note | -| ↳ `guid` | string | Unique identifier of the note | -| ↳ `title` | string | Title of the note | -| ↳ `content` | string | ENML content of the note | -| ↳ `notebookGuid` | string | GUID of the containing notebook | -| ↳ `tagNames` | array | Tag names applied to the note | -| ↳ `created` | number | Creation timestamp in milliseconds | -| ↳ `updated` | number | Last updated timestamp in milliseconds | - -### Evernote Create Notebook - -Create a new notebook in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `name` | string | Yes | Name for the new notebook | -| `stack` | string | No | Stack name to group the notebook under | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `notebook` | object | The created notebook | -| ↳ `guid` | string | Notebook GUID | -| ↳ `name` | string | Notebook name | -| ↳ `defaultNotebook` | boolean | Whether this is the default notebook | -| ↳ `serviceCreated` | number | Creation timestamp in milliseconds | -| ↳ `serviceUpdated` | number | Last updated timestamp in milliseconds | -| ↳ `stack` | string | Notebook stack name | - -### Evernote Create Tag - -Create a new tag in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `name` | string | Yes | Name for the new tag | -| `parentGuid` | string | No | GUID of the parent tag for hierarchy | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `tag` | object | The created tag | -| ↳ `guid` | string | Tag GUID | -| ↳ `name` | string | Tag name | -| ↳ `parentGuid` | string | Parent tag GUID | -| ↳ `updateSequenceNum` | number | Update sequence number | - -### Evernote Delete Note - -Move a note to the trash in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `noteGuid` | string | Yes | GUID of the note to delete | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `success` | boolean | Whether the note was successfully deleted | -| `noteGuid` | string | GUID of the deleted note | - -### Evernote Get Note - -Retrieve a note from Evernote by its GUID - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `noteGuid` | string | Yes | GUID of the note to retrieve | -| `withContent` | boolean | No | Whether to include note content \(default: true\) | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `note` | object | The retrieved note | -| ↳ `guid` | string | Unique identifier of the note | -| ↳ `title` | string | Title of the note | -| ↳ `content` | string | ENML content of the note | -| ↳ `contentLength` | number | Length of the note content | -| ↳ `notebookGuid` | string | GUID of the containing notebook | -| ↳ `tagGuids` | array | GUIDs of tags on the note | -| ↳ `tagNames` | array | Names of tags on the note | -| ↳ `created` | number | Creation timestamp in milliseconds | -| ↳ `updated` | number | Last updated timestamp in milliseconds | -| ↳ `active` | boolean | Whether the note is active \(not in trash\) | - -### Evernote Get Notebook - -Retrieve a notebook from Evernote by its GUID - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `notebookGuid` | string | Yes | GUID of the notebook to retrieve | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `notebook` | object | The retrieved notebook | -| ↳ `guid` | string | Notebook GUID | -| ↳ `name` | string | Notebook name | -| ↳ `defaultNotebook` | boolean | Whether this is the default notebook | -| ↳ `serviceCreated` | number | Creation timestamp in milliseconds | -| ↳ `serviceUpdated` | number | Last updated timestamp in milliseconds | -| ↳ `stack` | string | Notebook stack name | - -### Evernote List Notebooks - -List all notebooks in an Evernote account - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `notebooks` | array | List of notebooks | - -### Evernote List Tags - -List all tags in an Evernote account - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `tags` | array | List of tags | - -### Evernote Search Notes - -Search for notes in Evernote using the Evernote search grammar - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `query` | string | Yes | Search query using Evernote search grammar \(e.g., "tag:work intitle:meeting"\) | -| `notebookGuid` | string | No | Restrict search to a specific notebook by GUID | -| `offset` | number | No | Starting index for results \(default: 0\) | -| `maxNotes` | number | No | Maximum number of notes to return \(default: 25\) | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `totalNotes` | number | Total number of matching notes | -| `notes` | array | List of matching note metadata | - -### Evernote Update Note - -Update an existing note in Evernote - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Evernote developer token | -| `noteGuid` | string | Yes | GUID of the note to update | -| `title` | string | No | New title for the note | -| `content` | string | No | New content for the note \(plain text or ENML\) | -| `notebookGuid` | string | No | GUID of the notebook to move the note to | -| `tagNames` | string | No | Comma-separated list of tag names \(replaces existing tags\) | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `note` | object | The updated note | -| ↳ `guid` | string | Unique identifier of the note | -| ↳ `title` | string | Title of the note | -| ↳ `content` | string | ENML content of the note | -| ↳ `notebookGuid` | string | GUID of the containing notebook | -| ↳ `tagNames` | array | Tag names on the note | -| ↳ `created` | number | Creation timestamp in milliseconds | -| ↳ `updated` | number | Last updated timestamp in milliseconds | - - diff --git a/apps/docs/content/docs/en/integrations/meta.json b/apps/docs/content/docs/en/integrations/meta.json index c88cf373a73..5c4b4374bf6 100644 --- a/apps/docs/content/docs/en/integrations/meta.json +++ b/apps/docs/content/docs/en/integrations/meta.json @@ -73,7 +73,6 @@ "enrich", "enrichment", "enrow", - "evernote", "exa", "extend", "fathom", diff --git a/apps/docs/content/docs/en/knowledgebase/connectors.mdx b/apps/docs/content/docs/en/knowledgebase/connectors.mdx index 76e0e23a8ef..659fd5dae8d 100644 --- a/apps/docs/content/docs/en/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/en/knowledgebase/connectors.mdx @@ -12,9 +12,9 @@ Connectors continuously sync documents from external services into your knowledg ## Available Connectors - + -Sim ships with 61 built-in connectors: +Sim ships with 60 built-in connectors: | Category | Connectors | |----------|-----------| @@ -28,7 +28,7 @@ Sim ships with 61 built-in connectors: | **Support** | Intercom, ServiceNow, Zendesk, Zoho Desk | | **Incident Management** | incident.io, Rootly, PagerDuty | | **Data** | Airtable | -| **Note-taking** | Evernote, Obsidian | +| **Note-taking** | Obsidian | | **Meetings** | Zoom, Google Meet, Gong, Grain, Granola, Fathom, Fireflies | | **Recruiting** | Greenhouse, Ashby | | **Compliance** | Google Vault | @@ -48,7 +48,6 @@ Other connectors use **API keys** or **personal access tokens** instead. The set | Connector | Where to get the key | |-----------|---------------------| -| **Evernote** | Developer Token (starts with `S=`) from your Evernote account settings | | **Obsidian** | Install the [Local REST API](https://github.com/coddingtonbear/obsidian-local-rest-api) plugin, then copy the key from its settings | | **Fireflies** | Generate from the Integrations page in your Fireflies account | | **Typeform** | Personal access token from your Typeform account settings | @@ -57,7 +56,7 @@ Other connectors use **API keys** or **personal access tokens** instead. The set | **Amazon S3** | Secret Access Key (the Access Key ID, region, and bucket are entered as config fields) | | **Sentry** | Auth token with `project:read` and `event:read` scopes | | **PagerDuty** | REST API key from Integrations → API Access Keys | -| **SFTP** | Password or unencrypted private key (host, port, username, and root path are entered as config fields) | +| **SFTP** | Password or unencrypted private key, plus a required SHA-256 host key fingerprint (host, port, username, and root path are entered as config fields) | | **Mintlify** | API key — optional for public documentation sites, which sync from `llms.txt` | diff --git a/apps/sim/app/api/tools/docusign/route.ts b/apps/sim/app/api/tools/docusign/route.ts index e36b6cee009..0f21e39b413 100644 --- a/apps/sim/app/api/tools/docusign/route.ts +++ b/apps/sim/app/api/tools/docusign/route.ts @@ -13,6 +13,7 @@ import { readResponseToBufferWithLimit, } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' @@ -79,7 +80,7 @@ async function resolveAccount( signal?: AbortSignal ): Promise { const response = await fetchDocusign( - 'https://account-d.docusign.com/oauth/userinfo', + getDocusignOAuthUrl('/oauth/userinfo'), { headers: { Authorization: `Bearer ${accessToken}` }, }, diff --git a/apps/sim/app/api/tools/evernote/copy-note/route.ts b/apps/sim/app/api/tools/evernote/copy-note/route.ts deleted file mode 100644 index cddd0c2d404..00000000000 --- a/apps/sim/app/api/tools/evernote/copy-note/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteCopyNoteContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { copyNote } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteCopyNoteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteCopyNoteContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, noteGuid, toNotebookGuid } = parsed.data.body - const note = await copyNote(apiKey, noteGuid, toNotebookGuid) - - return NextResponse.json({ - success: true, - output: { note }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to copy note', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/create-note/route.ts b/apps/sim/app/api/tools/evernote/create-note/route.ts deleted file mode 100644 index 3e8aa3f1e97..00000000000 --- a/apps/sim/app/api/tools/evernote/create-note/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteCreateNoteContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createNote } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteCreateNoteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteCreateNoteContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, title, content, notebookGuid, tagNames } = parsed.data.body - const parsedTags = tagNames - ? (() => { - const tags = - typeof tagNames === 'string' - ? tagNames - .split(',') - .map((t: string) => t.trim()) - .filter(Boolean) - : tagNames - return tags.length > 0 ? tags : undefined - })() - : undefined - - const note = await createNote(apiKey, title, content, notebookGuid || undefined, parsedTags) - - return NextResponse.json({ - success: true, - output: { note }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to create note', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/create-notebook/route.ts b/apps/sim/app/api/tools/evernote/create-notebook/route.ts deleted file mode 100644 index c8a5ddd9c7d..00000000000 --- a/apps/sim/app/api/tools/evernote/create-notebook/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteCreateNotebookContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createNotebook } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteCreateNotebookAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteCreateNotebookContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, name, stack } = parsed.data.body - const notebook = await createNotebook(apiKey, name, stack || undefined) - - return NextResponse.json({ - success: true, - output: { notebook }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to create notebook', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/create-tag/route.ts b/apps/sim/app/api/tools/evernote/create-tag/route.ts deleted file mode 100644 index d0bd66517c2..00000000000 --- a/apps/sim/app/api/tools/evernote/create-tag/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteCreateTagContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createTag } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteCreateTagAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteCreateTagContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, name, parentGuid } = parsed.data.body - const tag = await createTag(apiKey, name, parentGuid || undefined) - - return NextResponse.json({ - success: true, - output: { tag }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to create tag', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/delete-note/route.ts b/apps/sim/app/api/tools/evernote/delete-note/route.ts deleted file mode 100644 index c1d4d6f4e9c..00000000000 --- a/apps/sim/app/api/tools/evernote/delete-note/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteDeleteNoteContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { deleteNote } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteDeleteNoteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteDeleteNoteContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, noteGuid } = parsed.data.body - await deleteNote(apiKey, noteGuid) - - return NextResponse.json({ - success: true, - output: { - success: true, - noteGuid, - }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to delete note', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/get-note/route.ts b/apps/sim/app/api/tools/evernote/get-note/route.ts deleted file mode 100644 index 5947a28ed98..00000000000 --- a/apps/sim/app/api/tools/evernote/get-note/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteGetNoteContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getNote } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteGetNoteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteGetNoteContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, noteGuid, withContent } = parsed.data.body - const note = await getNote(apiKey, noteGuid, withContent ?? true) - - return NextResponse.json({ - success: true, - output: { note }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to get note', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/get-notebook/route.ts b/apps/sim/app/api/tools/evernote/get-notebook/route.ts deleted file mode 100644 index 08bb24f26c0..00000000000 --- a/apps/sim/app/api/tools/evernote/get-notebook/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteGetNotebookContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getNotebook } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteGetNotebookAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteGetNotebookContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, notebookGuid } = parsed.data.body - const notebook = await getNotebook(apiKey, notebookGuid) - - return NextResponse.json({ - success: true, - output: { notebook }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to get notebook', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/lib/client.ts b/apps/sim/app/api/tools/evernote/lib/client.ts deleted file mode 100644 index 795f61a8a40..00000000000 --- a/apps/sim/app/api/tools/evernote/lib/client.ts +++ /dev/null @@ -1,799 +0,0 @@ -/** - * Evernote API client using Thrift binary protocol over HTTP. - * Implements only the NoteStore methods needed for the integration. - */ - -import { - ThriftReader, - ThriftWriter, - TYPE_BOOL, - TYPE_I32, - TYPE_I64, - TYPE_LIST, - TYPE_STRING, - TYPE_STRUCT, -} from './thrift' - -export interface EvernoteNotebook { - guid: string - name: string - defaultNotebook: boolean - serviceCreated: number | null - serviceUpdated: number | null - stack: string | null -} - -export interface EvernoteNote { - guid: string - title: string - content: string | null - contentLength: number | null - created: number | null - updated: number | null - deleted: number | null - active: boolean - notebookGuid: string | null - tagGuids: string[] - tagNames: string[] -} - -interface EvernoteNoteMetadata { - guid: string - title: string | null - contentLength: number | null - created: number | null - updated: number | null - notebookGuid: string | null - tagGuids: string[] -} - -export interface EvernoteTag { - guid: string - name: string - parentGuid: string | null - updateSequenceNum: number | null -} - -export interface EvernoteSearchResult { - startIndex: number - totalNotes: number - notes: EvernoteNoteMetadata[] -} - -/** Extract shard ID from an Evernote developer token */ -function extractShardId(token: string): string { - const match = token.match(/S=s(\d+)/) - if (!match) { - throw new Error('Invalid Evernote token format: cannot extract shard ID') - } - return `s${match[1]}` -} - -/** Get the NoteStore URL for the given token */ -function getNoteStoreUrl(token: string): string { - const shardId = extractShardId(token) - const host = token.includes(':Sandbox') ? 'sandbox.evernote.com' : 'www.evernote.com' - return `https://${host}/shard/${shardId}/notestore` -} - -/** Make a Thrift RPC call to the NoteStore */ -async function callNoteStore(token: string, writer: ThriftWriter): Promise { - const url = getNoteStoreUrl(token) - const body = writer.toBuffer() - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-thrift', - Accept: 'application/x-thrift', - }, - body: new Uint8Array(body), - }) - - if (!response.ok) { - throw new Error(`Evernote API HTTP error: ${response.status} ${response.statusText}`) - } - - const arrayBuffer = await response.arrayBuffer() - const reader = new ThriftReader(arrayBuffer) - const msg = reader.readMessageBegin() - - if (reader.isException(msg.type)) { - const ex = reader.readException() - throw new Error(`Evernote API error: ${ex.message}`) - } - - return reader -} - -/** Check for Evernote-specific exceptions in the response struct. Returns true if handled. */ -function checkEvernoteException(reader: ThriftReader, fieldId: number, fieldType: number): boolean { - if (fieldId === 1 && fieldType === TYPE_STRUCT) { - let message = '' - let errorCode = 0 - reader.readStruct((r, fid, ftype) => { - if (fid === 1 && ftype === TYPE_I32) { - errorCode = r.readI32() - } else if (fid === 2 && ftype === TYPE_STRING) { - message = r.readString() - } else { - r.skip(ftype) - } - }) - throw new Error(`Evernote error (${errorCode}): ${message}`) - } - if (fieldId === 2 && fieldType === TYPE_STRUCT) { - let message = '' - let errorCode = 0 - reader.readStruct((r, fid, ftype) => { - if (fid === 1 && ftype === TYPE_I32) { - errorCode = r.readI32() - } else if (fid === 2 && ftype === TYPE_STRING) { - message = r.readString() - } else { - r.skip(ftype) - } - }) - throw new Error(`Evernote system error (${errorCode}): ${message}`) - } - if (fieldId === 3 && fieldType === TYPE_STRUCT) { - let identifier = '' - let key = '' - reader.readStruct((r, fid, ftype) => { - if (fid === 1 && ftype === TYPE_STRING) { - identifier = r.readString() - } else if (fid === 2 && ftype === TYPE_STRING) { - key = r.readString() - } else { - r.skip(ftype) - } - }) - throw new Error(`Evernote not found: ${identifier}${key ? ` (${key})` : ''}`) - } - return false -} - -function readNotebook(reader: ThriftReader): EvernoteNotebook { - const notebook: EvernoteNotebook = { - guid: '', - name: '', - defaultNotebook: false, - serviceCreated: null, - serviceUpdated: null, - stack: null, - } - - reader.readStruct((r, fieldId, fieldType) => { - switch (fieldId) { - case 1: - if (fieldType === TYPE_STRING) notebook.guid = r.readString() - else r.skip(fieldType) - break - case 2: - if (fieldType === TYPE_STRING) notebook.name = r.readString() - else r.skip(fieldType) - break - case 4: - if (fieldType === TYPE_BOOL) notebook.defaultNotebook = r.readBool() - else r.skip(fieldType) - break - case 5: - if (fieldType === TYPE_I64) notebook.serviceCreated = Number(r.readI64()) - else r.skip(fieldType) - break - case 6: - if (fieldType === TYPE_I64) notebook.serviceUpdated = Number(r.readI64()) - else r.skip(fieldType) - break - case 9: - if (fieldType === TYPE_STRING) notebook.stack = r.readString() - else r.skip(fieldType) - break - default: - r.skip(fieldType) - } - }) - - return notebook -} - -function readNote(reader: ThriftReader): EvernoteNote { - const note: EvernoteNote = { - guid: '', - title: '', - content: null, - contentLength: null, - created: null, - updated: null, - deleted: null, - active: true, - notebookGuid: null, - tagGuids: [], - tagNames: [], - } - - reader.readStruct((r, fieldId, fieldType) => { - switch (fieldId) { - case 1: - if (fieldType === TYPE_STRING) note.guid = r.readString() - else r.skip(fieldType) - break - case 2: - if (fieldType === TYPE_STRING) note.title = r.readString() - else r.skip(fieldType) - break - case 3: - if (fieldType === TYPE_STRING) note.content = r.readString() - else r.skip(fieldType) - break - case 5: - if (fieldType === TYPE_I32) note.contentLength = r.readI32() - else r.skip(fieldType) - break - case 6: - if (fieldType === TYPE_I64) note.created = Number(r.readI64()) - else r.skip(fieldType) - break - case 7: - if (fieldType === TYPE_I64) note.updated = Number(r.readI64()) - else r.skip(fieldType) - break - case 8: - if (fieldType === TYPE_I64) note.deleted = Number(r.readI64()) - else r.skip(fieldType) - break - case 9: - if (fieldType === TYPE_BOOL) note.active = r.readBool() - else r.skip(fieldType) - break - case 11: - if (fieldType === TYPE_STRING) note.notebookGuid = r.readString() - else r.skip(fieldType) - break - case 12: - if (fieldType === TYPE_LIST) { - const { size } = r.readListBegin() - for (let i = 0; i < size; i++) { - note.tagGuids.push(r.readString()) - } - } else { - r.skip(fieldType) - } - break - case 15: - if (fieldType === TYPE_LIST) { - const { size } = r.readListBegin() - for (let i = 0; i < size; i++) { - note.tagNames.push(r.readString()) - } - } else { - r.skip(fieldType) - } - break - default: - r.skip(fieldType) - } - }) - - return note -} - -function readTag(reader: ThriftReader): EvernoteTag { - const tag: EvernoteTag = { - guid: '', - name: '', - parentGuid: null, - updateSequenceNum: null, - } - - reader.readStruct((r, fieldId, fieldType) => { - switch (fieldId) { - case 1: - if (fieldType === TYPE_STRING) tag.guid = r.readString() - else r.skip(fieldType) - break - case 2: - if (fieldType === TYPE_STRING) tag.name = r.readString() - else r.skip(fieldType) - break - case 3: - if (fieldType === TYPE_STRING) tag.parentGuid = r.readString() - else r.skip(fieldType) - break - case 4: - if (fieldType === TYPE_I32) tag.updateSequenceNum = r.readI32() - else r.skip(fieldType) - break - default: - r.skip(fieldType) - } - }) - - return tag -} - -function readNoteMetadata(reader: ThriftReader): EvernoteNoteMetadata { - const meta: EvernoteNoteMetadata = { - guid: '', - title: null, - contentLength: null, - created: null, - updated: null, - notebookGuid: null, - tagGuids: [], - } - - reader.readStruct((r, fieldId, fieldType) => { - switch (fieldId) { - case 1: - if (fieldType === TYPE_STRING) meta.guid = r.readString() - else r.skip(fieldType) - break - case 2: - if (fieldType === TYPE_STRING) meta.title = r.readString() - else r.skip(fieldType) - break - case 5: - if (fieldType === TYPE_I32) meta.contentLength = r.readI32() - else r.skip(fieldType) - break - case 6: - if (fieldType === TYPE_I64) meta.created = Number(r.readI64()) - else r.skip(fieldType) - break - case 7: - if (fieldType === TYPE_I64) meta.updated = Number(r.readI64()) - else r.skip(fieldType) - break - case 11: - if (fieldType === TYPE_STRING) meta.notebookGuid = r.readString() - else r.skip(fieldType) - break - case 12: - if (fieldType === TYPE_LIST) { - const { size } = r.readListBegin() - for (let i = 0; i < size; i++) { - meta.tagGuids.push(r.readString()) - } - } else { - r.skip(fieldType) - } - break - default: - r.skip(fieldType) - } - }) - - return meta -} - -export async function listNotebooks(token: string): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('listNotebooks', 0) - writer.writeStringField(1, token) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - const notebooks: EvernoteNotebook[] = [] - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_LIST) { - const { size } = r.readListBegin() - for (let i = 0; i < size; i++) { - notebooks.push(readNotebook(r)) - } - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - return notebooks -} - -export async function getNote( - token: string, - guid: string, - withContent = true -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('getNote', 0) - writer.writeStringField(1, token) - writer.writeStringField(2, guid) - writer.writeBoolField(3, withContent) - writer.writeBoolField(4, false) - writer.writeBoolField(5, false) - writer.writeBoolField(6, false) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let note: EvernoteNote | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - note = readNote(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!note) { - throw new Error('No note returned from Evernote API') - } - - return note -} - -/** Wrap content in ENML if it's not already */ -function wrapInEnml(content: string): string { - if (content.includes('/g, '>') - .replace(/\n/g, '
') - return `${escaped}` -} - -export async function createNote( - token: string, - title: string, - content: string, - notebookGuid?: string, - tagNames?: string[] -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('createNote', 0) - writer.writeStringField(1, token) - - writer.writeFieldBegin(TYPE_STRUCT, 2) - writer.writeStringField(2, title) - writer.writeStringField(3, wrapInEnml(content)) - if (notebookGuid) { - writer.writeStringField(11, notebookGuid) - } - if (tagNames && tagNames.length > 0) { - writer.writeStringListField(15, tagNames) - } - writer.writeFieldStop() - - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let note: EvernoteNote | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - note = readNote(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!note) { - throw new Error('No note returned from Evernote API') - } - - return note -} - -export async function updateNote( - token: string, - guid: string, - title?: string, - content?: string, - notebookGuid?: string, - tagNames?: string[] -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('updateNote', 0) - writer.writeStringField(1, token) - - writer.writeFieldBegin(TYPE_STRUCT, 2) - writer.writeStringField(1, guid) - if (title !== undefined) { - writer.writeStringField(2, title) - } - if (content !== undefined) { - writer.writeStringField(3, wrapInEnml(content)) - } - if (notebookGuid !== undefined) { - writer.writeStringField(11, notebookGuid) - } - if (tagNames !== undefined) { - writer.writeStringListField(15, tagNames) - } - writer.writeFieldStop() - - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let note: EvernoteNote | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - note = readNote(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!note) { - throw new Error('No note returned from Evernote API') - } - - return note -} - -export async function deleteNote(token: string, guid: string): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('deleteNote', 0) - writer.writeStringField(1, token) - writer.writeStringField(2, guid) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let usn = 0 - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_I32) { - usn = r.readI32() - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - return usn -} - -export async function searchNotes( - token: string, - query: string, - notebookGuid?: string, - offset = 0, - maxNotes = 25 -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('findNotesMetadata', 0) - writer.writeStringField(1, token) - - // NoteFilter (field 2) - writer.writeFieldBegin(TYPE_STRUCT, 2) - if (query) { - writer.writeStringField(3, query) - } - if (notebookGuid) { - writer.writeStringField(4, notebookGuid) - } - writer.writeFieldStop() - - // offset (field 3) - writer.writeI32Field(3, offset) - // maxNotes (field 4) - writer.writeI32Field(4, maxNotes) - - // NotesMetadataResultSpec (field 5) - writer.writeFieldBegin(TYPE_STRUCT, 5) - writer.writeBoolField(2, true) // includeTitle - writer.writeBoolField(5, true) // includeContentLength - writer.writeBoolField(6, true) // includeCreated - writer.writeBoolField(7, true) // includeUpdated - writer.writeBoolField(11, true) // includeNotebookGuid - writer.writeBoolField(12, true) // includeTagGuids - writer.writeFieldStop() - - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - const result: EvernoteSearchResult = { - startIndex: 0, - totalNotes: 0, - notes: [], - } - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - r.readStruct((r2, fid2, ftype2) => { - switch (fid2) { - case 1: - if (ftype2 === TYPE_I32) result.startIndex = r2.readI32() - else r2.skip(ftype2) - break - case 2: - if (ftype2 === TYPE_I32) result.totalNotes = r2.readI32() - else r2.skip(ftype2) - break - case 3: - if (ftype2 === TYPE_LIST) { - const { size } = r2.readListBegin() - for (let i = 0; i < size; i++) { - result.notes.push(readNoteMetadata(r2)) - } - } else { - r2.skip(ftype2) - } - break - default: - r2.skip(ftype2) - } - }) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - return result -} - -export async function getNotebook(token: string, guid: string): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('getNotebook', 0) - writer.writeStringField(1, token) - writer.writeStringField(2, guid) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let notebook: EvernoteNotebook | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - notebook = readNotebook(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!notebook) { - throw new Error('No notebook returned from Evernote API') - } - - return notebook -} - -export async function createNotebook( - token: string, - name: string, - stack?: string -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('createNotebook', 0) - writer.writeStringField(1, token) - - writer.writeFieldBegin(TYPE_STRUCT, 2) - writer.writeStringField(2, name) - if (stack) { - writer.writeStringField(9, stack) - } - writer.writeFieldStop() - - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let notebook: EvernoteNotebook | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - notebook = readNotebook(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!notebook) { - throw new Error('No notebook returned from Evernote API') - } - - return notebook -} - -export async function listTags(token: string): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('listTags', 0) - writer.writeStringField(1, token) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - const tags: EvernoteTag[] = [] - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_LIST) { - const { size } = r.readListBegin() - for (let i = 0; i < size; i++) { - tags.push(readTag(r)) - } - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - return tags -} - -export async function createTag( - token: string, - name: string, - parentGuid?: string -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('createTag', 0) - writer.writeStringField(1, token) - - writer.writeFieldBegin(TYPE_STRUCT, 2) - writer.writeStringField(2, name) - if (parentGuid) { - writer.writeStringField(3, parentGuid) - } - writer.writeFieldStop() - - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let tag: EvernoteTag | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - tag = readTag(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!tag) { - throw new Error('No tag returned from Evernote API') - } - - return tag -} - -export async function copyNote( - token: string, - noteGuid: string, - toNotebookGuid: string -): Promise { - const writer = new ThriftWriter() - writer.writeMessageBegin('copyNote', 0) - writer.writeStringField(1, token) - writer.writeStringField(2, noteGuid) - writer.writeStringField(3, toNotebookGuid) - writer.writeFieldStop() - - const reader = await callNoteStore(token, writer) - let note: EvernoteNote | null = null - - reader.readStruct((r, fieldId, fieldType) => { - if (fieldId === 0 && fieldType === TYPE_STRUCT) { - note = readNote(r) - } else { - if (!checkEvernoteException(r, fieldId, fieldType)) { - r.skip(fieldType) - } - } - }) - - if (!note) { - throw new Error('No note returned from Evernote API') - } - - return note -} diff --git a/apps/sim/app/api/tools/evernote/lib/thrift.ts b/apps/sim/app/api/tools/evernote/lib/thrift.ts deleted file mode 100644 index 811bf3462a3..00000000000 --- a/apps/sim/app/api/tools/evernote/lib/thrift.ts +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Minimal Thrift binary protocol encoder/decoder for Evernote API. - * Supports only the types needed for NoteStore operations. - */ - -const THRIFT_VERSION_1 = 0x80010000 -const MESSAGE_CALL = 1 -const MESSAGE_EXCEPTION = 3 - -const TYPE_STOP = 0 -const TYPE_BOOL = 2 -const TYPE_I32 = 8 -const TYPE_I64 = 10 -const TYPE_STRING = 11 -const TYPE_STRUCT = 12 -const TYPE_LIST = 15 - -export class ThriftWriter { - private buffer: number[] = [] - - writeMessageBegin(name: string, seqId: number): void { - this.writeI32(THRIFT_VERSION_1 | MESSAGE_CALL) - this.writeString(name) - this.writeI32(seqId) - } - - writeFieldBegin(type: number, id: number): void { - this.buffer.push(type) - this.writeI16(id) - } - - writeFieldStop(): void { - this.buffer.push(TYPE_STOP) - } - - writeString(value: string): void { - const encoded = new TextEncoder().encode(value) - this.writeI32(encoded.length) - for (const byte of encoded) { - this.buffer.push(byte) - } - } - - writeBool(value: boolean): void { - this.buffer.push(value ? 1 : 0) - } - - writeI16(value: number): void { - this.buffer.push((value >> 8) & 0xff) - this.buffer.push(value & 0xff) - } - - writeI32(value: number): void { - this.buffer.push((value >> 24) & 0xff) - this.buffer.push((value >> 16) & 0xff) - this.buffer.push((value >> 8) & 0xff) - this.buffer.push(value & 0xff) - } - - writeI64(value: bigint): void { - const buf = new ArrayBuffer(8) - const view = new DataView(buf) - view.setBigInt64(0, value, false) - for (let i = 0; i < 8; i++) { - this.buffer.push(view.getUint8(i)) - } - } - - writeStringField(id: number, value: string): void { - this.writeFieldBegin(TYPE_STRING, id) - this.writeString(value) - } - - writeBoolField(id: number, value: boolean): void { - this.writeFieldBegin(TYPE_BOOL, id) - this.writeBool(value) - } - - writeI32Field(id: number, value: number): void { - this.writeFieldBegin(TYPE_I32, id) - this.writeI32(value) - } - - writeStringListField(id: number, values: string[]): void { - this.writeFieldBegin(TYPE_LIST, id) - this.buffer.push(TYPE_STRING) - this.writeI32(values.length) - for (const v of values) { - this.writeString(v) - } - } - - toBuffer(): Buffer { - return Buffer.from(this.buffer) - } -} - -export class ThriftReader { - private view: DataView - private pos = 0 - - constructor(buffer: ArrayBuffer) { - this.view = new DataView(buffer) - } - - readMessageBegin(): { name: string; type: number; seqId: number } { - const versionAndType = this.readI32() - const version = versionAndType & 0xffff0000 - if (version !== (THRIFT_VERSION_1 | 0)) { - throw new Error(`Unsupported Thrift version: 0x${version.toString(16)}`) - } - const type = versionAndType & 0x000000ff - const name = this.readString() - const seqId = this.readI32() - return { name, type, seqId } - } - - readFieldBegin(): { type: number; id: number } { - const type = this.view.getUint8(this.pos++) - if (type === TYPE_STOP) { - return { type: TYPE_STOP, id: 0 } - } - const id = this.view.getInt16(this.pos, false) - this.pos += 2 - return { type, id } - } - - readString(): string { - const length = this.readI32() - const bytes = new Uint8Array(this.view.buffer, this.pos, length) - this.pos += length - return new TextDecoder().decode(bytes) - } - - readBool(): boolean { - return this.view.getUint8(this.pos++) !== 0 - } - - readI32(): number { - const value = this.view.getInt32(this.pos, false) - this.pos += 4 - return value - } - - readI64(): bigint { - const value = this.view.getBigInt64(this.pos, false) - this.pos += 8 - return value - } - - readBinary(): Uint8Array { - const length = this.readI32() - const bytes = new Uint8Array(this.view.buffer, this.pos, length) - this.pos += length - return bytes - } - - readListBegin(): { elementType: number; size: number } { - const elementType = this.view.getUint8(this.pos++) - const size = this.readI32() - return { elementType, size } - } - - /** Skip a value of the given Thrift type */ - skip(type: number): void { - switch (type) { - case TYPE_BOOL: - this.pos += 1 - break - case 6: // I16 - this.pos += 2 - break - case 3: // BYTE - this.pos += 1 - break - case TYPE_I32: - this.pos += 4 - break - case TYPE_I64: - case 4: // DOUBLE - this.pos += 8 - break - case TYPE_STRING: { - const len = this.readI32() - this.pos += len - break - } - case TYPE_STRUCT: - this.skipStruct() - break - case TYPE_LIST: - case 14: { - // SET - const { elementType, size } = this.readListBegin() - for (let i = 0; i < size; i++) { - this.skip(elementType) - } - break - } - case 13: { - // MAP - const keyType = this.view.getUint8(this.pos++) - const valueType = this.view.getUint8(this.pos++) - const count = this.readI32() - for (let i = 0; i < count; i++) { - this.skip(keyType) - this.skip(valueType) - } - break - } - default: - throw new Error(`Cannot skip unknown Thrift type: ${type}`) - } - } - - private skipStruct(): void { - for (;;) { - const { type } = this.readFieldBegin() - if (type === TYPE_STOP) break - this.skip(type) - } - } - - /** Read struct fields, calling the handler for each field */ - readStruct(handler: (reader: ThriftReader, fieldId: number, fieldType: number) => void): void { - for (;;) { - const { type, id } = this.readFieldBegin() - if (type === TYPE_STOP) break - handler(this, id, type) - } - } - - /** Check if this is an exception response */ - isException(messageType: number): boolean { - return messageType === MESSAGE_EXCEPTION - } - - /** Read a Thrift application exception */ - readException(): { message: string; type: number } { - let message = '' - let type = 0 - this.readStruct((reader, fieldId, fieldType) => { - if (fieldId === 1 && fieldType === TYPE_STRING) { - message = reader.readString() - } else if (fieldId === 2 && fieldType === TYPE_I32) { - type = reader.readI32() - } else { - reader.skip(fieldType) - } - }) - return { message, type } - } -} - -export { TYPE_BOOL, TYPE_I32, TYPE_I64, TYPE_LIST, TYPE_STRING, TYPE_STRUCT } diff --git a/apps/sim/app/api/tools/evernote/list-notebooks/route.ts b/apps/sim/app/api/tools/evernote/list-notebooks/route.ts deleted file mode 100644 index bf55d7a87cb..00000000000 --- a/apps/sim/app/api/tools/evernote/list-notebooks/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteListNotebooksContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { listNotebooks } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteListNotebooksAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteListNotebooksContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey } = parsed.data.body - const notebooks = await listNotebooks(apiKey) - - return NextResponse.json({ - success: true, - output: { notebooks }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to list notebooks', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/list-tags/route.ts b/apps/sim/app/api/tools/evernote/list-tags/route.ts deleted file mode 100644 index ffbe255c16d..00000000000 --- a/apps/sim/app/api/tools/evernote/list-tags/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteListTagsContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { listTags } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteListTagsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteListTagsContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey } = parsed.data.body - const tags = await listTags(apiKey) - - return NextResponse.json({ - success: true, - output: { tags }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to list tags', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/search-notes/route.ts b/apps/sim/app/api/tools/evernote/search-notes/route.ts deleted file mode 100644 index ae9a3120249..00000000000 --- a/apps/sim/app/api/tools/evernote/search-notes/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteSearchNotesContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { searchNotes } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteSearchNotesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteSearchNotesContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, query, notebookGuid, offset, maxNotes } = parsed.data.body - const clampedMaxNotes = Math.min(Math.max(Number(maxNotes) || 25, 1), 250) - - const result = await searchNotes( - apiKey, - query, - notebookGuid || undefined, - Number(offset), - clampedMaxNotes - ) - - return NextResponse.json({ - success: true, - output: { - totalNotes: result.totalNotes, - notes: result.notes, - }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to search notes', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/evernote/update-note/route.ts b/apps/sim/app/api/tools/evernote/update-note/route.ts deleted file mode 100644 index f1d816328ef..00000000000 --- a/apps/sim/app/api/tools/evernote/update-note/route.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { evernoteUpdateNoteContract } from '@/lib/api/contracts/tools/evernote' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { updateNote } from '@/app/api/tools/evernote/lib/client' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('EvernoteUpdateNoteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - evernoteUpdateNoteContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ success: false, error: 'Invalid request' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { apiKey, noteGuid, title, content, notebookGuid, tagNames } = parsed.data.body - const parsedTags = tagNames - ? (() => { - const tags = - typeof tagNames === 'string' - ? tagNames - .split(',') - .map((t: string) => t.trim()) - .filter(Boolean) - : tagNames - return tags.length > 0 ? tags : undefined - })() - : undefined - - const note = await updateNote( - apiKey, - noteGuid, - title || undefined, - content || undefined, - notebookGuid || undefined, - parsedTags - ) - - return NextResponse.json({ - success: true, - output: { note }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Failed to update note', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/monday/boards/route.ts b/apps/sim/app/api/tools/monday/boards/route.ts index a3de9b41989..bc877e2fcea 100644 --- a/apps/sim/app/api/tools/monday/boards/route.ts +++ b/apps/sim/app/api/tools/monday/boards/route.ts @@ -6,6 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' +import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' export const dynamic = 'force-dynamic' @@ -72,13 +73,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let page = 1 for (; page <= MAX_MONDAY_PAGES; page++) { - const response = await fetch('https://api.monday.com/v2', { + const response = await fetch(MONDAY_API_URL, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: accessToken, - 'API-Version': '2024-10', - }, + headers: mondayHeaders(accessToken), body: JSON.stringify({ query: `{ boards(limit: ${MONDAY_BOARDS_LIMIT}, page: ${page}, state: active) { id name } }`, }), diff --git a/apps/sim/app/api/tools/monday/groups/route.ts b/apps/sim/app/api/tools/monday/groups/route.ts index 3492f448564..de80412e1cf 100644 --- a/apps/sim/app/api/tools/monday/groups/route.ts +++ b/apps/sim/app/api/tools/monday/groups/route.ts @@ -7,6 +7,7 @@ import { validateMondayNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' +import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' export const dynamic = 'force-dynamic' @@ -65,13 +66,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const response = await fetch('https://api.monday.com/v2', { + const response = await fetch(MONDAY_API_URL, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: accessToken, - 'API-Version': '2024-10', - }, + headers: mondayHeaders(accessToken), body: JSON.stringify({ query: `{ boards(ids: [${boardIdValidation.sanitized}]) { groups { id title } } }`, }), diff --git a/apps/sim/blocks/blocks/evernote.ts b/apps/sim/blocks/blocks/evernote.ts deleted file mode 100644 index 3a4e94915ba..00000000000 --- a/apps/sim/blocks/blocks/evernote.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { EvernoteIcon } from '@/components/icons' -import type { BlockConfig, BlockMeta } from '@/blocks/types' -import { AuthMode, IntegrationType } from '@/blocks/types' - -export const EvernoteBlock: BlockConfig = { - type: 'evernote', - name: 'Evernote', - description: 'Manage notes, notebooks, and tags in Evernote', - longDescription: - 'Integrate with Evernote to manage notes, notebooks, and tags. Create, read, update, copy, search, and delete notes. Create and list notebooks and tags.', - docsLink: 'https://docs.sim.ai/integrations/evernote', - category: 'tools', - integrationType: IntegrationType.Documents, - bgColor: '#FFFFFF', - icon: EvernoteIcon, - canvasPresentation: { - defaultTitle: 'Evernote', - sentences: { - byOperation: { - create_note: [ - { text: 'Create note', field: 'title', core: true }, - { text: 'in notebook', field: 'notebookGuid' }, - { text: ', tagged', field: 'tagNames' }, - ], - get_note: [{ text: 'Read note', field: 'noteGuid', core: true }], - update_note: [ - { text: 'Update note', field: 'noteGuid', core: true }, - { text: ', renaming to', field: 'updateTitle' }, - { text: ', tagged', field: 'tagNames' }, - ], - delete_note: [{ text: 'Move note', field: 'noteGuid', core: true, after: 'to the trash' }], - copy_note: [ - { text: 'Copy note', field: 'noteGuid', core: true }, - { text: 'into notebook', field: 'toNotebookGuid' }, - ], - search_notes: [ - { text: 'Search notes for', field: 'query', core: true }, - { text: ', within notebook', field: 'notebookGuid' }, - ], - get_notebook: [{ text: 'Read notebook', field: 'notebookGuid', core: true }], - create_notebook: [ - { text: 'Create notebook', field: 'notebookName', core: true }, - { text: 'in stack', field: 'stack' }, - ], - list_notebooks: ['List all notebooks'], - create_tag: [ - { text: 'Create tag', field: 'tagName', core: true }, - { text: 'under parent tag', field: 'parentGuid' }, - ], - list_tags: ['List all tags'], - }, - }, - }, - authMode: AuthMode.ApiKey, - - subBlocks: [ - { - id: 'operation', - title: 'Operation', - type: 'dropdown', - options: [ - { label: 'Create Note', id: 'create_note' }, - { label: 'Get Note', id: 'get_note' }, - { label: 'Update Note', id: 'update_note' }, - { label: 'Delete Note', id: 'delete_note' }, - { label: 'Copy Note', id: 'copy_note' }, - { label: 'Search Notes', id: 'search_notes' }, - { label: 'Get Notebook', id: 'get_notebook' }, - { label: 'Create Notebook', id: 'create_notebook' }, - { label: 'List Notebooks', id: 'list_notebooks' }, - { label: 'Create Tag', id: 'create_tag' }, - { label: 'List Tags', id: 'list_tags' }, - ], - value: () => 'create_note', - }, - { - id: 'apiKey', - title: 'Developer Token', - type: 'short-input', - password: true, - placeholder: 'Enter your Evernote developer token', - required: true, - }, - { - id: 'title', - title: 'Title', - type: 'short-input', - placeholder: 'Note title', - condition: { field: 'operation', value: 'create_note' }, - required: { field: 'operation', value: 'create_note' }, - }, - { - id: 'content', - title: 'Content', - type: 'long-input', - placeholder: 'Note content (plain text or ENML)', - condition: { field: 'operation', value: 'create_note' }, - required: { field: 'operation', value: 'create_note' }, - }, - { - id: 'noteGuid', - title: 'Note GUID', - type: 'short-input', - placeholder: 'Enter the note GUID', - condition: { - field: 'operation', - value: ['get_note', 'update_note', 'delete_note', 'copy_note'], - }, - required: { - field: 'operation', - value: ['get_note', 'update_note', 'delete_note', 'copy_note'], - }, - }, - { - id: 'updateTitle', - title: 'New Title', - type: 'short-input', - placeholder: 'New title (leave empty to keep current)', - condition: { field: 'operation', value: 'update_note' }, - }, - { - id: 'updateContent', - title: 'New Content', - type: 'long-input', - placeholder: 'New content (leave empty to keep current)', - condition: { field: 'operation', value: 'update_note' }, - }, - { - id: 'toNotebookGuid', - title: 'Destination Notebook GUID', - type: 'short-input', - placeholder: 'GUID of the destination notebook', - condition: { field: 'operation', value: 'copy_note' }, - required: { field: 'operation', value: 'copy_note' }, - }, - { - id: 'query', - title: 'Search Query', - type: 'short-input', - placeholder: 'e.g., "tag:work intitle:meeting"', - condition: { field: 'operation', value: 'search_notes' }, - required: { field: 'operation', value: 'search_notes' }, - }, - { - id: 'notebookGuid', - title: 'Notebook GUID', - type: 'short-input', - placeholder: 'Notebook GUID', - condition: { - field: 'operation', - value: ['create_note', 'update_note', 'search_notes', 'get_notebook'], - }, - required: { field: 'operation', value: 'get_notebook' }, - }, - { - id: 'notebookName', - title: 'Notebook Name', - type: 'short-input', - placeholder: 'Name for the new notebook', - condition: { field: 'operation', value: 'create_notebook' }, - required: { field: 'operation', value: 'create_notebook' }, - }, - { - id: 'stack', - title: 'Stack', - type: 'short-input', - placeholder: 'Stack name (optional)', - condition: { field: 'operation', value: 'create_notebook' }, - mode: 'advanced', - }, - { - id: 'tagName', - title: 'Tag Name', - type: 'short-input', - placeholder: 'Name for the new tag', - condition: { field: 'operation', value: 'create_tag' }, - required: { field: 'operation', value: 'create_tag' }, - }, - { - id: 'parentGuid', - title: 'Parent Tag GUID', - type: 'short-input', - placeholder: 'Parent tag GUID (optional)', - condition: { field: 'operation', value: 'create_tag' }, - mode: 'advanced', - }, - { - id: 'tagNames', - title: 'Tags', - type: 'short-input', - placeholder: 'Comma-separated tags (e.g., "work, meeting, urgent")', - condition: { field: 'operation', value: ['create_note', 'update_note'] }, - mode: 'advanced', - }, - { - id: 'maxNotes', - title: 'Max Results', - type: 'short-input', - placeholder: '25', - condition: { field: 'operation', value: 'search_notes' }, - mode: 'advanced', - }, - { - id: 'offset', - title: 'Offset', - type: 'short-input', - placeholder: '0', - condition: { field: 'operation', value: 'search_notes' }, - mode: 'advanced', - }, - { - id: 'withContent', - title: 'Include Content', - type: 'dropdown', - options: [ - { label: 'Yes', id: 'true' }, - { label: 'No', id: 'false' }, - ], - value: () => 'true', - condition: { field: 'operation', value: 'get_note' }, - mode: 'advanced', - }, - ], - - tools: { - access: [ - 'evernote_copy_note', - 'evernote_create_note', - 'evernote_create_notebook', - 'evernote_create_tag', - 'evernote_delete_note', - 'evernote_get_note', - 'evernote_get_notebook', - 'evernote_list_notebooks', - 'evernote_list_tags', - 'evernote_search_notes', - 'evernote_update_note', - ], - config: { - tool: (params) => `evernote_${params.operation}`, - params: (params) => { - const { operation, apiKey, ...rest } = params - - switch (operation) { - case 'create_note': - return { - apiKey, - title: rest.title, - content: rest.content, - notebookGuid: rest.notebookGuid || undefined, - tagNames: rest.tagNames || undefined, - } - case 'get_note': - return { - apiKey, - noteGuid: rest.noteGuid, - withContent: rest.withContent !== 'false', - } - case 'update_note': - return { - apiKey, - noteGuid: rest.noteGuid, - title: rest.updateTitle || undefined, - content: rest.updateContent || undefined, - notebookGuid: rest.notebookGuid || undefined, - tagNames: rest.tagNames || undefined, - } - case 'delete_note': - return { - apiKey, - noteGuid: rest.noteGuid, - } - case 'copy_note': - return { - apiKey, - noteGuid: rest.noteGuid, - toNotebookGuid: rest.toNotebookGuid, - } - case 'search_notes': - return { - apiKey, - query: rest.query, - notebookGuid: rest.notebookGuid || undefined, - offset: rest.offset ? Number(rest.offset) : 0, - maxNotes: rest.maxNotes ? Number(rest.maxNotes) : 25, - } - case 'get_notebook': - return { - apiKey, - notebookGuid: rest.notebookGuid, - } - case 'create_notebook': - return { - apiKey, - name: rest.notebookName, - stack: rest.stack || undefined, - } - case 'list_notebooks': - return { apiKey } - case 'create_tag': - return { - apiKey, - name: rest.tagName, - parentGuid: rest.parentGuid || undefined, - } - case 'list_tags': - return { apiKey } - default: - return { apiKey } - } - }, - }, - }, - - inputs: { - apiKey: { type: 'string', description: 'Evernote developer token' }, - operation: { type: 'string', description: 'Operation to perform' }, - title: { type: 'string', description: 'Note title' }, - content: { type: 'string', description: 'Note content' }, - noteGuid: { type: 'string', description: 'Note GUID' }, - updateTitle: { type: 'string', description: 'New note title' }, - updateContent: { type: 'string', description: 'New note content' }, - toNotebookGuid: { type: 'string', description: 'Destination notebook GUID' }, - query: { type: 'string', description: 'Search query' }, - notebookGuid: { type: 'string', description: 'Notebook GUID' }, - notebookName: { type: 'string', description: 'Notebook name' }, - stack: { type: 'string', description: 'Notebook stack name' }, - tagName: { type: 'string', description: 'Tag name' }, - parentGuid: { type: 'string', description: 'Parent tag GUID' }, - tagNames: { type: 'string', description: 'Comma-separated tag names' }, - maxNotes: { type: 'string', description: 'Maximum number of results' }, - offset: { type: 'string', description: 'Starting index for results' }, - withContent: { type: 'string', description: 'Whether to include note content' }, - }, - - outputs: { - note: { type: 'json', description: 'Note data' }, - notebook: { type: 'json', description: 'Notebook data' }, - notebooks: { type: 'json', description: 'List of notebooks' }, - tag: { type: 'json', description: 'Tag data' }, - tags: { type: 'json', description: 'List of tags' }, - totalNotes: { type: 'number', description: 'Total number of matching notes' }, - notes: { type: 'json', description: 'List of note metadata' }, - success: { type: 'boolean', description: 'Whether the operation succeeded' }, - noteGuid: { type: 'string', description: 'GUID of the affected note' }, - }, -} - -export const EvernoteBlockMeta = { - tags: ['note-taking', 'knowledge-base'], - url: 'https://evernote.com', - templates: [ - { - icon: EvernoteIcon, - title: 'Evernote to knowledge base sync', - prompt: - 'Build a workflow that syncs Evernote notebooks into a knowledge base on a schedule so all notes and clipped web pages become searchable by an agent.', - modules: ['knowledge-base', 'agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'research'], - }, - { - icon: EvernoteIcon, - title: 'Evernote weekly summary', - prompt: - 'Create a scheduled weekly workflow that summarizes new Evernote notes by tag, writes the summary as a Markdown file, and emails it to the user as a knowledge digest.', - modules: ['scheduled', 'agent', 'files', 'workflows'], - category: 'productivity', - tags: ['individual', 'reporting'], - alsoIntegrations: ['gmail'], - }, - { - icon: EvernoteIcon, - title: 'Evernote action-item extractor', - prompt: - 'Build a workflow that searches Evernote for recently created notes, extracts action items and due dates with an agent, and creates a matching task in Asana for each.', - modules: ['agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'automation'], - alsoIntegrations: ['asana'], - }, - { - icon: EvernoteIcon, - title: 'Evernote research collector', - prompt: - 'Create a workflow that takes web clippings saved to Evernote, classifies by topic, and writes structured rows to a research table for downstream analysis.', - modules: ['tables', 'agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'research'], - }, - { - icon: EvernoteIcon, - title: 'Evernote tag auto-organizer', - prompt: - 'Build a workflow that scans new Evernote notes, suggests and applies tags based on content, and writes the tag changes to an audit log for review.', - modules: ['agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'automation'], - }, - { - icon: EvernoteIcon, - title: 'Evernote to Notion migrator', - prompt: - 'Create a workflow that imports an Evernote notebook into Notion as pages in a chosen database, preserving formatting, attachments, and tags.', - modules: ['files', 'agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'sync'], - alsoIntegrations: ['notion'], - }, - { - icon: EvernoteIcon, - title: 'Evernote research-assistant agent', - prompt: - 'Build an agent that searches across the user’s Evernote notebooks for grounded answers with citations, and saves the answer plus sources back as a new Evernote note.', - modules: ['agent', 'workflows'], - category: 'productivity', - tags: ['individual', 'research'], - }, - ], - skills: [ - { - name: 'create-evernote-note', - description: 'Create a new Evernote note with a title, content, tags, and target notebook.', - content: - '# Create Evernote Note\n\nSave new content as a note in Evernote.\n\n## Steps\n1. Confirm the note title and content. Plain text is fine; the content is stored as ENML.\n2. Choose the Create Note operation. To file it in a specific notebook, resolve the notebook GUID with List Notebooks and pass it.\n3. Add comma-separated tag names so the note is findable later.\n\n## Output\nReturn the new note GUID, its title, and the notebook and tags it was saved under.', - }, - { - name: 'search-evernote-notes', - description: - 'Search Evernote for notes matching a query and return their titles and metadata.', - content: - '# Search Evernote Notes\n\nFind notes across Evernote using its search grammar.\n\n## Steps\n1. Build a query using Evernote search syntax — e.g., tag:work, intitle:meeting, notebook scoping, or plain keywords.\n2. Run Search Notes. Scope to a notebook GUID when the location is known, and set max results and offset to page through matches.\n3. For any note you need the body of, call Get Note with its GUID and include content.\n\n## Output\nReturn the matching notes with title, GUID, and notebook, plus the total match count. If a note body is needed, include its retrieved content.', - }, - { - name: 'extract-note-action-items', - description: 'Read recent Evernote notes and extract action items, owners, and due dates.', - content: - '# Extract Note Action Items\n\nPull tasks out of meeting notes or research notes in Evernote.\n\n## Steps\n1. Use Search Notes to find the relevant recent notes (e.g., by tag or notebook).\n2. For each match, call Get Note with content to read the full body.\n3. Identify action items, the responsible owner, and any due dates mentioned in the text.\n\n## Output\nReturn a structured list of action items, each with its owner, due date if stated, and a link back to the source note GUID. Flag items with no clear owner.', - }, - { - name: 'organize-notes-with-tags', - description: 'Create tags and apply them to Evernote notes to keep them organized.', - content: - '# Organize Notes with Tags\n\nKeep Evernote notes structured by tagging them consistently.\n\n## Steps\n1. Call List Tags to see existing tags and avoid duplicates. Create any missing tag with Create Tag (optionally nested under a parent tag).\n2. For each note to organize, read it with Get Note if needed, decide the right tags from its content, and apply them via Update Note with the tag names.\n3. Keep tag names consistent in casing and wording across notes.\n\n## Output\nReturn each note GUID with the tags applied and note any new tags that were created.', - }, - ], -} as const satisfies BlockMeta diff --git a/apps/sim/blocks/registry-maps.ts b/apps/sim/blocks/registry-maps.ts index 629df4273d7..3c526800afa 100644 --- a/apps/sim/blocks/registry-maps.ts +++ b/apps/sim/blocks/registry-maps.ts @@ -74,7 +74,6 @@ import { EnrichBlock, EnrichBlockMeta } from '@/blocks/blocks/enrich' import { EnrichmentBlock, EnrichmentBlockMeta } from '@/blocks/blocks/enrichment' import { EnrowBlock, EnrowBlockMeta } from '@/blocks/blocks/enrow' import { EvaluatorBlock } from '@/blocks/blocks/evaluator' -import { EvernoteBlock, EvernoteBlockMeta } from '@/blocks/blocks/evernote' import { ExaBlock, ExaBlockMeta } from '@/blocks/blocks/exa' import { ExtendBlock, ExtendBlockMeta, ExtendV2Block } from '@/blocks/blocks/extend' import { FathomBlock, FathomBlockMeta } from '@/blocks/blocks/fathom' @@ -436,7 +435,6 @@ export const BLOCK_REGISTRY: Record = { enrichment: EnrichmentBlock, enrow: EnrowBlock, evaluator: EvaluatorBlock, - evernote: EvernoteBlock, exa: ExaBlock, extend: ExtendBlock, extend_v2: ExtendV2Block, @@ -769,7 +767,6 @@ export const BLOCK_META_REGISTRY: Record = { enrich: EnrichBlockMeta, enrichment: EnrichmentBlockMeta, enrow: EnrowBlockMeta, - evernote: EvernoteBlockMeta, exa: ExaBlockMeta, extend: ExtendBlockMeta, fathom: FathomBlockMeta, diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index e40d9af2efd..5d9b3efae5c 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -2769,14 +2769,6 @@ export function ExtendIcon(props: SVGProps) { ) } -export function EvernoteIcon(props: SVGProps) { - return ( - - - - ) -} - export function ElevenLabsIcon(props: SVGProps) { return ( ): string { + const stable = value.filename ?? value.name ?? value.text ?? value.label + if (typeof stable === 'string' || typeof stable === 'number' || typeof stable === 'boolean') { + return String(stable) + } + if (typeof value.id === 'string') return value.id + if (typeof value.email === 'string') return value.email + return '' +} + +/** Renders an object- or array-valued cell, dropping items that render to nothing. */ +function formatCellValue(value: object): string { + if (!Array.isArray(value)) return formatCellObject(value as Record) + return value + .map((item) => + typeof item === 'object' && item !== null + ? formatCellObject(item as Record) + : String(item) + ) + .filter((item) => item.length > 0) + .join(', ') +} + /** * Flattens a record's fields into a plain-text representation. * Each field is rendered as "Field Name: value" on its own line. */ -function recordToPlainText( - fields: Record, - fieldNames?: Map -): string { +function recordToPlainText(fields: Record): string { const lines: string[] = [] for (const [key, value] of Object.entries(fields)) { if (value == null) continue - const displayName = fieldNames?.get(key) ?? key - if (Array.isArray(value)) { - // Attachments or linked records - const items = value.map((v) => { - if (typeof v === 'object' && v !== null) { - const obj = v as Record - return (obj.url as string) || (obj.name as string) || JSON.stringify(v) - } - return String(v) - }) - lines.push(`${displayName}: ${items.join(', ')}`) - } else if (typeof value === 'object') { - lines.push(`${displayName}: ${JSON.stringify(value)}`) + if (typeof value === 'object') { + const rendered = formatCellValue(value) + if (!rendered) continue + lines.push(`${key}: ${rendered}`) } else { - lines.push(`${displayName}: ${String(value)}`) + lines.push(`${key}: ${String(value)}`) } } return lines.join('\n') } +/** + * Airtable long-text cells are unbounded, so a title derived from one is capped + * to keep document titles readable in the knowledge base UI. + */ +const MAX_TITLE_LENGTH = 200 + +/** Field names tried, in order, when no `titleField` is configured or it is empty. */ +const TITLE_FALLBACK_FIELDS = ['Name', 'Title', 'name', 'title', 'Summary', 'summary'] as const + +/** Renders a candidate cell as a title, or null when it holds nothing usable. */ +function renderTitle(value: unknown): string | null { + if (value == null) return null + const rendered = typeof value === 'object' ? formatCellValue(value).trim() : String(value).trim() + if (!rendered) return null + return rendered.length > MAX_TITLE_LENGTH ? `${rendered.slice(0, MAX_TITLE_LENGTH)}…` : rendered +} + /** * Extracts a human-readable title from a record's fields. * Prefers the configured title field, then falls back to common field names. */ function extractTitle(fields: Record, titleField?: string): string { - if (titleField && fields[titleField] != null) { - return String(fields[titleField]) + if (titleField) { + const fromConfigured = renderTitle(fields[titleField]) + if (fromConfigured) return fromConfigured } - const candidates = ['Name', 'Title', 'name', 'title', 'Summary', 'summary'] - for (const candidate of candidates) { - if (fields[candidate] != null) { - return String(fields[candidate]) - } + for (const candidate of TITLE_FALLBACK_FIELDS) { + const fromCandidate = renderTitle(fields[candidate]) + if (fromCandidate) return fromCandidate } for (const value of Object.values(fields)) { - if (typeof value === 'string' && value.trim()) { - return value.length > 80 ? `${value.slice(0, 80)}…` : value - } + if (typeof value !== 'string') continue + const rendered = renderTitle(value) + if (rendered) return rendered } return 'Untitled' } @@ -72,23 +110,62 @@ function parseCursor(cursor?: string): string | undefined { return cursor } +function readConfigString(sourceConfig: Record, key: string): string | undefined { + const raw = sourceConfig[key] + if (typeof raw !== 'string') return undefined + const trimmed = raw.trim() + return trimmed.length > 0 ? trimmed : undefined +} + +/** Parses the optional `maxRecords` cap; Airtable requires a positive integer. */ +function readMaxRecords(sourceConfig: Record): number { + const raw = sourceConfig.maxRecords + if (raw == null || raw === '') return 0 + const parsed = Math.floor(Number(raw)) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +} + export const airtableConnector: ConnectorConfig = { ...airtableConnectorMeta, + /** + * Lists records from `GET /v0/{baseId}/{tableIdOrName}`. + * + * Scope semantics that matter for deletion reconciliation: + * - A configured `view` is an intentional scope filter: the source set *is* + * the view. A record leaving the view is indistinguishable from a deleted + * record over this API, and both should drop out of the knowledge base, so + * `listingCapped` is deliberately NOT set for view scoping. + * - `maxRecords` truncates a listing that still has records behind it, so it + * sets `listingCapped` to suppress hard deletion of the records beyond the + * cap. + */ listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, syncContext?: Record ): Promise => { - const baseId = sourceConfig.baseId as string - const tableIdOrName = sourceConfig.tableIdOrName as string - const viewId = sourceConfig.viewId as string | undefined - const titleField = sourceConfig.titleField as string | undefined - const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0 + const baseId = readConfigString(sourceConfig, 'baseId') + const tableIdOrName = readConfigString(sourceConfig, 'tableIdOrName') + if (!baseId || !tableIdOrName) { + throw new Error('Airtable connector is missing baseId or tableIdOrName') + } + const viewId = readConfigString(sourceConfig, 'viewId') + const titleField = readConfigString(sourceConfig, 'titleField') + const maxRecords = readMaxRecords(sourceConfig) + + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 - const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName, syncContext) + const tableId = await resolveTableId(accessToken, baseId, tableIdOrName, syncContext) + /** + * `pageSize` is held at the documented maximum for every request of a sync. + * Airtable already stops pagination itself once `maxRecords` is reached, and + * its `offset` is an opaque iterator token whose validity across a changed + * `pageSize` is undocumented — so shrinking the last page would buy nothing + * and risk breaking iteration mid-sync. + */ const params = new URLSearchParams() params.append('pageSize', String(PAGE_SIZE)) if (viewId) params.append('view', viewId) @@ -97,8 +174,9 @@ export const airtableConnector: ConnectorConfig = { const offset = parseCursor(cursor) if (offset) params.append('offset', offset) + const encodedBase = encodeURIComponent(baseId) const encodedTable = encodeURIComponent(tableIdOrName) - const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?${params.toString()}` + const url = `${AIRTABLE_API}/${encodedBase}/${encodedTable}?${params.toString()}` logger.info(`Listing records from ${baseId}/${tableIdOrName}`, { offset: offset ?? 'none', @@ -118,41 +196,65 @@ export const airtableConnector: ConnectorConfig = { status: response.status, error: errorText, }) + /** + * Airtable expires the list iterator after a period of inactivity and + * answers a stale `offset` with 422 LIST_RECORDS_ITERATOR_NOT_AVAILABLE. + * Throwing aborts the sync before reconciliation, which is the safe + * outcome — the next run restarts iteration from the beginning. + */ throw new Error(`Failed to list Airtable records: ${response.status}`) } const data = (await response.json()) as { - records: AirtableRecord[] + records?: AirtableRecord[] offset?: string } - const records = data.records || [] + const records = data.records ?? [] const documents: ExternalDocument[] = await Promise.all( - records.map((record) => - recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames) - ) + records.map((record) => recordToDocument(record, baseId, tableId, titleField)) ) + const totalFetched = prevFetched + documents.length + if (syncContext) syncContext.totalDocsFetched = totalFetched + const nextOffset = data.offset + const hitLimit = maxRecords > 0 && totalFetched >= maxRecords + /** + * Airtable enforces `maxRecords` itself — "pagination will stop once you've + * reached this maximum" — but does not document whether it still returns an + * `offset` at that point, so an exhausted source and a capped one cannot be + * told apart here. Flagged conservatively: a capped listing must never let + * the engine hard-delete the records the cap hid. The cost is that deletion + * reconciliation only runs for a capped source on an explicit full resync. + */ + if (hitLimit && syncContext) syncContext.listingCapped = true + return { documents, - nextCursor: nextOffset ? `offset:${nextOffset}` : undefined, - hasMore: Boolean(nextOffset), + nextCursor: !hitLimit && nextOffset ? `offset:${nextOffset}` : undefined, + hasMore: !hitLimit && Boolean(nextOffset), } }, getDocument: async ( accessToken: string, sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { - const baseId = sourceConfig.baseId as string - const tableIdOrName = sourceConfig.tableIdOrName as string - const titleField = sourceConfig.titleField as string | undefined + const baseId = readConfigString(sourceConfig, 'baseId') + const tableIdOrName = readConfigString(sourceConfig, 'tableIdOrName') + /** A broken config is not evidence the record is gone, so it must not read as absence. */ + if (!baseId || !tableIdOrName) { + throw new Error('Airtable connector is missing baseId or tableIdOrName') + } + const titleField = readConfigString(sourceConfig, 'titleField') - const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName) + const tableId = await resolveTableId(accessToken, baseId, tableIdOrName, syncContext) + const encodedBase = encodeURIComponent(baseId) const encodedTable = encodeURIComponent(tableIdOrName) - const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}` + const url = `${AIRTABLE_API}/${encodedBase}/${encodedTable}/${encodeURIComponent(externalId)}` const response = await fetchWithRetry(url, { method: 'GET', @@ -167,32 +269,36 @@ export const airtableConnector: ConnectorConfig = { } const record = (await response.json()) as AirtableRecord - return recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames) + return recordToDocument(record, baseId, tableId, titleField) }, validateConfig: async ( accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const baseId = sourceConfig.baseId as string - const tableIdOrName = sourceConfig.tableIdOrName as string + const baseId = readConfigString(sourceConfig, 'baseId') + const tableIdOrName = readConfigString(sourceConfig, 'tableIdOrName') if (!baseId || !tableIdOrName) { return { valid: false, error: 'Base ID and table name are required' } } - if (baseId && !baseId.startsWith('app')) { + if (!baseId.startsWith('app')) { return { valid: false, error: 'Base ID should start with "app"' } } - const maxRecords = sourceConfig.maxRecords as string | undefined - if (maxRecords && (Number.isNaN(Number(maxRecords)) || Number(maxRecords) <= 0)) { - return { valid: false, error: 'Max records must be a positive number' } + const rawMaxRecords = sourceConfig.maxRecords + if (rawMaxRecords != null && rawMaxRecords !== '') { + const parsed = Number(rawMaxRecords) + if (!Number.isInteger(parsed) || parsed <= 0) { + return { valid: false, error: 'Max records must be a positive whole number' } + } } try { + const encodedBase = encodeURIComponent(baseId) const encodedTable = encodeURIComponent(tableIdOrName) - const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1` + const url = `${AIRTABLE_API}/${encodedBase}/${encodedTable}?pageSize=1` const response = await fetchWithRetry( url, { @@ -215,9 +321,9 @@ export const airtableConnector: ConnectorConfig = { return { valid: false, error: `Airtable API error: ${response.status} - ${errorText}` } } - const viewId = sourceConfig.viewId as string | undefined + const viewId = readConfigString(sourceConfig, 'viewId') if (viewId) { - const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}` + const viewUrl = `${AIRTABLE_API}/${encodedBase}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}` const viewResponse = await fetchWithRetry( viewUrl, { @@ -258,20 +364,24 @@ interface AirtableRecord { /** * Converts an Airtable record to an ExternalDocument. + * + * `tableId` is the `tbl…` identifier when it could be resolved — Airtable + * record deep links are only valid with the table ID, never the table name. */ async function recordToDocument( record: AirtableRecord, baseId: string, - tableIdOrName: string, - titleField: string | undefined, - fieldNames: Map + tableId: string | undefined, + titleField: string | undefined ): Promise { - const plainText = recordToPlainText(record.fields, fieldNames) + const fields = record.fields ?? {} + const plainText = recordToPlainText(fields) const contentHash = await computeContentHash(plainText) - const title = extractTitle(record.fields, titleField) + const title = extractTitle(fields, titleField) - const encodedTable = encodeURIComponent(tableIdOrName) - const sourceUrl = `https://airtable.com/${baseId}/${encodedTable}/${record.id}` + const sourceUrl = tableId + ? `https://airtable.com/${baseId}/${tableId}/${record.id}` + : `https://airtable.com/${baseId}` return { externalId: record.id, @@ -287,21 +397,29 @@ async function recordToDocument( } /** - * Fetches the table schema to build a field ID → field name mapping. + * Resolves the configured table reference to its `tbl…` ID via the Meta API + * (`GET /v0/meta/bases/{baseId}/tables`, `schema.bases:read`), cached in + * `syncContext` so a sync spends at most one extra request against the 5 req/s + * per-base rate limit. Returns undefined when the schema is unreadable — the + * record link degrades to the base link rather than emitting a broken URL. */ -async function fetchFieldNames( +async function resolveTableId( accessToken: string, baseId: string, tableIdOrName: string, syncContext?: Record -): Promise> { - const cacheKey = `fieldNames:${baseId}/${tableIdOrName}` - if (syncContext?.[cacheKey]) return syncContext[cacheKey] as Map +): Promise { + if (tableIdOrName.startsWith('tbl')) return tableIdOrName - const fieldNames = new Map() + const cacheKey = `tableId:${baseId}/${tableIdOrName}` + if (syncContext && cacheKey in syncContext) { + return syncContext[cacheKey] as string | undefined + } + + let resolved: string | undefined try { - const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables` + const url = `${AIRTABLE_API}/meta/bases/${encodeURIComponent(baseId)}/tables` const response = await fetchWithRetry(url, { method: 'GET', headers: { @@ -309,31 +427,20 @@ async function fetchFieldNames( }, }) - if (!response.ok) { - logger.warn('Failed to fetch Airtable schema, using raw field keys', { + if (response.ok) { + const data = (await response.json()) as { tables?: { id: string; name: string }[] } + resolved = (data.tables ?? []).find((t) => t.name === tableIdOrName)?.id + } else { + logger.warn('Failed to fetch Airtable base schema; record links will point at the base', { status: response.status, }) - return fieldNames - } - - const data = (await response.json()) as { - tables: { id: string; name: string; fields: { id: string; name: string; type: string }[] }[] - } - - const table = data.tables.find((t) => t.id === tableIdOrName || t.name === tableIdOrName) - - if (table) { - for (const field of table.fields) { - fieldNames.set(field.id, field.name) - fieldNames.set(field.name, field.name) - } } } catch (error) { - logger.warn('Error fetching Airtable schema', { + logger.warn('Error fetching Airtable base schema', { error: toError(error).message, }) } - if (syncContext) syncContext[cacheKey] = fieldNames - return fieldNames + if (syncContext) syncContext[cacheKey] = resolved + return resolved } diff --git a/apps/sim/connectors/airtable/meta.ts b/apps/sim/connectors/airtable/meta.ts index dc4dd4a393d..3044ceca757 100644 --- a/apps/sim/connectors/airtable/meta.ts +++ b/apps/sim/connectors/airtable/meta.ts @@ -5,7 +5,7 @@ export const airtableConnectorMeta: ConnectorMeta = { id: 'airtable', name: 'Airtable', description: 'Sync records from an Airtable table', - version: '1.0.0', + version: '1.1.0', icon: AirtableIcon, auth: { diff --git a/apps/sim/connectors/asana/asana.test.ts b/apps/sim/connectors/asana/asana.test.ts index 950e505ba25..aa005fa3a30 100644 --- a/apps/sim/connectors/asana/asana.test.ts +++ b/apps/sim/connectors/asana/asana.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { asanaConnector, buildProjectsPath, + buildTasksPath, decideTaskCap, isActiveProject, isTaskUnderActiveProject, @@ -39,6 +40,27 @@ describe('buildProjectsPath', () => { }) }) +describe('buildTasksPath', () => { + it.concurrent('scopes the listing to the project with a page size', () => { + const path = buildTasksPath('p1', 100) + expect(path.startsWith('/tasks?')).toBe(true) + expect(path).toContain('project=p1') + expect(path).toContain('limit=100') + }) + + it.concurrent('omits the offset param on the first page', () => { + expect(buildTasksPath('p1', 100)).not.toContain('offset=') + }) + + it.concurrent('escapes the opaque offset token instead of interpolating it raw', () => { + expect(buildTasksPath('p1', 100, 'abc:def')).toContain('offset=abc%3Adef') + }) + + it.concurrent('keeps opt_fields separators as literal commas', () => { + expect(buildTasksPath('p1', 100)).toContain('opt_fields=name,notes,completed,modified_at') + }) +}) + describe('isActiveProject', () => { it.concurrent('keeps projects explicitly marked as not archived', () => { expect(isActiveProject({ gid: '1', name: 'Roadmap', archived: false })).toBe(true) @@ -376,6 +398,71 @@ describe('asanaConnector.listDocuments', () => { expect(syncContext.listingCapped).toBeUndefined() }) + it('drains several sparse projects in one call instead of one project per page', async () => { + mockFetch.mockImplementation(async (url) => { + if (url.includes('/projects')) { + return jsonResponse({ + data: [ + { gid: 'p1', name: 'A' }, + { gid: 'p2', name: 'B' }, + { gid: 'p3', name: 'C' }, + ], + next_page: null, + }) + } + return jsonResponse({ data: [], next_page: null }) + }) + + const result = await asanaConnector.listDocuments('token', { workspace: 'w1' }, undefined, {}) + + expect(requestedUrls().filter((url) => url.includes('/tasks?')).length).toBe(3) + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + }) + + it('stops after the per-call request budget and hands back a resumable cursor', async () => { + mockFetch.mockImplementation(async (url) => { + if (url.includes('/projects')) { + return jsonResponse({ + data: Array.from({ length: 40 }, (_, i) => ({ gid: `p${i}`, name: `P${i}` })), + next_page: null, + }) + } + return jsonResponse({ data: [], next_page: null }) + }) + + const syncContext: Record = {} + const result = await asanaConnector.listDocuments( + 'token', + { workspace: 'w1' }, + undefined, + syncContext + ) + + expect(requestedUrls().filter((url) => url.includes('/tasks?')).length).toBe(25) + expect(result.hasMore).toBe(true) + expect(JSON.parse(result.nextCursor as string)).toEqual({ projectIndex: 25 }) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('holds the requested page size constant regardless of how much cap is left', async () => { + mockFetch.mockImplementation(async (url) => { + if (url.includes('/projects')) { + return jsonResponse({ data: [{ gid: 'p1', name: 'Live' }], next_page: null }) + } + return jsonResponse({ + data: [{ gid: 't9', name: 'Nine', completed: false }], + next_page: null, + }) + }) + + await asanaConnector.listDocuments('token', { workspace: 'w1', maxTasks: '500' }, undefined, { + totalDocsFetched: 497, + }) + + expect(requestedUrls().find((url) => url.includes('/tasks?'))).toContain('limit=100') + }) + it('keeps syncing an explicitly pinned project without listing workspace projects', async () => { mockFetch.mockImplementation(async () => jsonResponse({ data: [{ gid: 't1', name: 'One', completed: false }], next_page: null }) diff --git a/apps/sim/connectors/asana/asana.ts b/apps/sim/connectors/asana/asana.ts index 9c32111beb4..576921e2882 100644 --- a/apps/sim/connectors/asana/asana.ts +++ b/apps/sim/connectors/asana/asana.ts @@ -9,8 +9,25 @@ const logger = createLogger('AsanaConnector') const ASANA_API = 'https://app.asana.com/api/1.0' -const TASK_OPT_FIELDS = - 'name,notes,completed,completed_at,modified_at,assignee.name,tags.name,permalink_url' +const TASK_OPT_FIELDS = 'name,notes,completed,modified_at,assignee.name,tags.name,permalink_url' + +/** + * Asana caps `limit` at 100 objects per page on every collection endpoint. + */ +const ASANA_MAX_PAGE_SIZE = 100 + +/** + * Upper bound on Asana task requests issued per `listDocuments` call. + * + * The connector walks one project at a time, so without this the choice is + * between one HTTP request per sync page — which burns the sync engine's + * `MAX_PAGES` budget on workspaces with more projects than that, silently + * setting `listingTruncated` and blocking deletion reconciliation forever — and + * an unbounded loop that fans a 5,000-project workspace into 5,000 sequential + * requests inside a single call. Batching a bounded number of projects per call + * keeps both the request budget and the per-call time budget in range. + */ +const MAX_TASK_REQUESTS_PER_PAGE = 25 /** * Asana API response shape for paginated endpoints. @@ -28,7 +45,6 @@ export interface AsanaTask { name: string notes?: string completed: boolean - completed_at?: string modified_at?: string assignee?: { name: string } tags?: { name: string }[] @@ -65,7 +81,9 @@ const PROJECT_OPT_FIELDS = 'gid,name,archived' * parent projects (with their archived flag) on top of the listing fields so * `isTaskUnderActiveProject` can run on the rehydrate path. `opt_fields` * supports dot paths, so `projects.archived` expands the compact project stubs - * with the field the listing filter relies on. + * with the field the listing filter relies on. `projects.gid` is deliberately + * not requested: Asana always returns the `gid` of included objects regardless + * of the field options. */ const TASK_DETAIL_OPT_FIELDS = `${TASK_OPT_FIELDS},projects.archived` @@ -95,6 +113,21 @@ export function buildProjectsPath(workspaceGid: string, offset?: string): string return `/projects?${params.toString()}&opt_fields=${PROJECT_OPT_FIELDS}` } +/** + * Builds the project task listing path. + * + * Mirrors `buildProjectsPath`: caller-supplied values (project gid, pagination + * offset) go through `URLSearchParams` so they are escaped, while `opt_fields` + * is appended raw to keep its separators literal commas. Asana offsets are + * opaque tokens it may change the encoding of, so they are never interpolated + * unescaped. + */ +export function buildTasksPath(projectGid: string, limit: number, offset?: string): string { + const params = new URLSearchParams({ project: projectGid, limit: String(limit) }) + if (offset) params.append('offset', offset) + return `/tasks?${params.toString()}&opt_fields=${TASK_OPT_FIELDS}` +} + /** * Keeps only projects that are still active. Asana regressed the server-side * `archived=false` filter once before (fixed 2024-11-06), so results are @@ -175,6 +208,20 @@ export function decideTaskCap( } } +/** + * Carries the HTTP status so callers can tell a genuinely missing task (404) from a + * transient fault (429, 5xx) that must not be swallowed. + */ +class AsanaApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'AsanaApiError' + } +} + /** * Makes a GET request to the Asana REST API. */ @@ -198,7 +245,7 @@ async function asanaGet( if (!response.ok) { const errorText = await response.text() logger.error('Asana API request failed', { status: response.status, path, error: errorText }) - throw new Error(`Asana API error: ${response.status}`) + throw new AsanaApiError(`Asana API error: ${response.status}`, response.status) } return (await response.json()) as T @@ -276,7 +323,15 @@ export const asanaConnector: ConnectorConfig = { const workspaceGid = sourceConfig.workspace as string const projectGid = (sourceConfig.project as string) || '' const maxTasks = sourceConfig.maxTasks ? Number(sourceConfig.maxTasks) : 0 - const pageSize = maxTasks > 0 ? Math.min(maxTasks, 100) : 100 + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + + /** + * Held constant across every request of a sync. Asana's `offset` is an + * opaque token and the docs do not say it survives a changed `limit`, while + * `decideTaskCap` already trims the cap exactly — so varying the page size + * per call would buy nothing and risk invalidating a mid-project offset. + */ + const pageSize = maxTasks > 0 ? Math.min(maxTasks, ASANA_MAX_PAGE_SIZE) : ASANA_MAX_PAGE_SIZE /** * Cursor format: @@ -322,13 +377,18 @@ export const asanaConnector: ConnectorConfig = { let nextCursor: string | undefined let hasMore = false - while (projectIndex < projectGids.length) { + for ( + let request = 0; + projectIndex < projectGids.length && + request < MAX_TASK_REQUESTS_PER_PAGE && + documents.length < pageSize; + request++ + ) { const currentProjectGid = projectGids[projectIndex] - const offsetParam = offset ? `&offset=${offset}` : '' const result = await asanaGet( accessToken, - `/tasks?project=${currentProjectGid}&opt_fields=${TASK_OPT_FIELDS}&limit=${pageSize}${offsetParam}` + buildTasksPath(currentProjectGid, pageSize, offset) ) for (const task of result.data) { @@ -353,22 +413,18 @@ export const asanaConnector: ConnectorConfig = { } if (result.next_page) { - nextCursor = JSON.stringify({ projectIndex, offset: result.next_page.offset }) + offset = result.next_page.offset + nextCursor = JSON.stringify({ projectIndex, offset }) hasMore = true - break + continue } projectIndex++ offset = undefined - - if (projectIndex < projectGids.length) { - nextCursor = JSON.stringify({ projectIndex, offset: undefined }) - hasMore = true - break - } + hasMore = projectIndex < projectGids.length + nextCursor = hasMore ? JSON.stringify({ projectIndex }) : undefined } - const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 const cap = decideTaskCap(maxTasks, previouslyFetched, documents.length, hasMore) if (cap.keepCount < documents.length) documents.splice(cap.keepCount) @@ -396,7 +452,7 @@ export const asanaConnector: ConnectorConfig = { try { const result = await asanaGet<{ data: AsanaTask }>( accessToken, - `/tasks/${externalId}?opt_fields=${TASK_DETAIL_OPT_FIELDS}` + `/tasks/${encodeURIComponent(externalId)}?opt_fields=${TASK_DETAIL_OPT_FIELDS}` ) const task = result.data @@ -418,6 +474,13 @@ export const asanaConnector: ConnectorConfig = { sourceUrl: task.permalink_url || undefined, contentHash: `asana:${task.gid}:${task.modified_at ?? ''}`, metadata: { + /** + * The listing reaches a task through exactly one project and records + * that gid, so the rehydrate path must record one too or the `project` + * tag would silently disappear on refresh. A pinned project wins; + * otherwise the task's first still-active project stands in. + */ + project: pinnedProjectGid ?? task.projects?.find((p) => p?.archived !== true)?.gid, assignee: task.assignee?.name, completed: task.completed, lastModified: task.modified_at, @@ -425,11 +488,20 @@ export const asanaConnector: ConnectorConfig = { }, } } catch (error) { + /** + * Only a 404 means the task is genuinely gone. Every other failure — 429, 5xx, + * network faults — is rethrown so the sync engine records a failed row and keeps + * the already-indexed task out of deletion reconciliation. + */ + if (error instanceof AsanaApiError && error.status === 404) { + logger.info('Asana task not found', { externalId }) + return null + } logger.error('Failed to get Asana task', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/ashby/ashby.ts b/apps/sim/connectors/ashby/ashby.ts index 7ab1eb39381..298bbd8c18e 100644 --- a/apps/sim/connectors/ashby/ashby.ts +++ b/apps/sim/connectors/ashby/ashby.ts @@ -19,6 +19,13 @@ const FEEDBACK_PER_PAGE = 100 */ const MAX_APPLICATIONS_FOR_FEEDBACK = 10 +/** + * Defensive page ceiling for the per-candidate note and feedback cursor loops. Ashby + * terminates them via `moreDataAvailable`, but a repeated cursor would otherwise spin + * forever inside a single `getDocument` call. + */ +const MAX_SUB_PAGES = 100 + type UnknownRecord = Record /** @@ -43,13 +50,29 @@ interface AshbyEnvelope { } /** - * Extracts a human-readable error message from an Ashby error envelope. Ashby returns - * errors as either `errorInfo.message` or an `errors` string array. + * Extracts a human-readable error message from an Ashby error envelope. The documented + * failure body is `{ success: false, errors: [{ message }] }`, but `errorInfo.message` + * and plain-string `errors` entries also occur, so all three are handled. Reading the + * object entry's `message` explicitly is what keeps it from stringifying to + * `[object Object]`. */ function ashbyErrorMessage(data: AshbyEnvelope, fallback: string): string { if (data.errorInfo?.message) return data.errorInfo.message if (Array.isArray(data.errors) && data.errors.length > 0) { - return data.errors.map((e) => String(e)).join('; ') + const messages = data.errors + .map((entry) => { + if (typeof entry === 'string') return entry.trim() + if (entry && typeof entry === 'object') { + const e = entry as UnknownRecord + const message = typeof e.message === 'string' ? e.message.trim() : '' + const parameter = typeof e.parameter === 'string' ? e.parameter.trim() : '' + if (message && parameter) return `${message} (${parameter})` + if (message) return message + } + return '' + }) + .filter(Boolean) + if (messages.length > 0) return messages.join('; ') } return fallback } @@ -214,21 +237,50 @@ interface AshbyFeedbackSummary { lines: string[] } +interface AshbyFeedbackField { + title: string + /** `selectableValues` stored value -> display label, for select-type fields. */ + labelByValue: Map +} + /** - * Collects `{ field.path -> field.title }` entries from a feedback form definition. - * Ashby's `formDefinition` exposes fields either flat under `fields[]` or grouped - * under `sections[].fields[]`, and individual entries are sometimes wrapped in a - * `{ field }` envelope — all variants are handled. + * Collects `{ field.path -> { title, labelByValue } }` entries from a feedback form + * definition. Ashby's `formDefinition` exposes fields either flat under `fields[]` or + * grouped under `sections[].fields[]`, and individual entries are sometimes wrapped in a + * `{ isRequired, field }` envelope — all variants are handled. + * + * Select-type fields (`ValueSelect`, `MultiValueSelect`, `Score`) return the stored + * option value in `submittedValues`, not its display label, so `selectableValues` + * (`[{ label, value }]`) is indexed here to render human-readable text. + * + * Ref: https://developers.ashbyhq.com/reference/applicationfeedbacklist */ -function collectFieldTitles(formDefinition: UnknownRecord | undefined): Map { - const titleByPath = new Map() - if (!formDefinition) return titleByPath +function collectFeedbackFields( + formDefinition: UnknownRecord | undefined +): Map { + const fieldByPath = new Map() + if (!formDefinition) return fieldByPath const addField = (entry: UnknownRecord): void => { const field = (entry?.field ?? entry) as UnknownRecord const path = field?.path as string | undefined - const title = (field?.title as string) || (field?.humanReadablePath as string) - if (path && title) titleByPath.set(path, title) + if (!path) return + + const title = (field?.title as string) || (field?.humanReadablePath as string) || path + const labelByValue = new Map() + if (Array.isArray(field?.selectableValues)) { + for (const option of field.selectableValues as UnknownRecord[]) { + const label = option?.label + const value = option?.value + const isScalar = + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' + if (typeof label === 'string' && label.trim() && isScalar) { + labelByValue.set(String(value), label.trim()) + } + } + } + + fieldByPath.set(path, { title, labelByValue }) } if (Array.isArray(formDefinition.fields)) { @@ -242,7 +294,7 @@ function collectFieldTitles(formDefinition: UnknownRecord | undefined): Map): string { + if (typeof value === 'string') { + const trimmed = value.trim() + return labelByValue?.get(trimmed) ?? labelByValue?.get(value) ?? trimmed + } + if (typeof value === 'number' || typeof value === 'boolean') { + return labelByValue?.get(String(value)) ?? String(value) + } if (Array.isArray(value)) { return value - .map((v) => renderFeedbackValue(v)) + .map((v) => renderFeedbackValue(v, labelByValue)) .filter(Boolean) .join(', ') } @@ -371,7 +430,7 @@ async function fetchAllNotes(accessToken: string, candidateId: string): Promise< let cursor: string | undefined let hasMore = true - while (hasMore) { + for (let page = 0; hasMore && page < MAX_SUB_PAGES; page++) { const body: UnknownRecord = { candidateId, limit: NOTES_PER_PAGE } if (cursor) body.cursor = cursor const data = await ashbyPost(accessToken, 'candidate.listNotes', body) @@ -381,6 +440,13 @@ async function fetchAllNotes(accessToken: string, candidateId: string): Promise< hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor) } + if (hasMore) { + logger.warn('Stopped paginating Ashby candidate notes at the page ceiling', { + candidateId, + notes: notes.length, + }) + } + return notes } @@ -396,7 +462,7 @@ async function fetchFeedbackForApplication( let cursor: string | undefined let hasMore = true - while (hasMore) { + for (let page = 0; hasMore && page < MAX_SUB_PAGES; page++) { const body: UnknownRecord = { applicationId, limit: FEEDBACK_PER_PAGE } if (cursor) body.cursor = cursor const data = await ashbyPost(accessToken, 'applicationFeedback.list', body) @@ -406,6 +472,13 @@ async function fetchFeedbackForApplication( hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor) } + if (hasMore) { + logger.warn('Stopped paginating Ashby application feedback at the page ceiling', { + applicationId, + submissions: feedback.length, + }) + } + return feedback } @@ -474,10 +547,15 @@ export const ashbyConnector: ConnectorConfig = { const prevFetched = (syncContext?.totalCandidatesFetched as number) ?? 0 if (maxCandidates > 0 && prevFetched >= maxCandidates) { - if (syncContext) syncContext.listingCapped = true return { documents: [], hasMore: false } } + /** + * `limit` is held constant for every request of a sync: Ashby's `cursor` is + * opaque and the docs do not say it survives a changed `limit`, and the cap + * is enforced below by trimming the page instead. + */ + const remaining = maxCandidates > 0 ? maxCandidates - prevFetched : Number.POSITIVE_INFINITY const body: UnknownRecord = { limit: CANDIDATES_PER_PAGE } if (cursor) body.cursor = cursor if (createdAfterMs !== undefined) body.createdAfter = createdAfterMs @@ -491,19 +569,28 @@ export const ashbyConnector: ConnectorConfig = { const results = Array.isArray(data.results) ? data.results : [] const candidates = results.map(mapCandidate).filter((c) => c.id) - let documents = candidates.map(candidateToStub) - if (maxCandidates > 0) { - const remaining = Math.max(0, maxCandidates - prevFetched) - if (documents.length > remaining) documents = documents.slice(0, remaining) - } + const stubs = candidates.map(candidateToStub) + const documents = stubs.length > remaining ? stubs.slice(0, remaining) : stubs + /** True when the cap hid candidates Ashby already returned on this very page. */ + const droppedInPage = documents.length < stubs.length const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalCandidatesFetched = totalFetched - const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates - if (hitLimit && syncContext) syncContext.listingCapped = true const nextCursor = data.nextCursor ?? undefined - const hasMore = !hitLimit && Boolean(data.moreDataAvailable) && Boolean(nextCursor) + const sourceHasMore = Boolean(data.moreDataAvailable) && Boolean(nextCursor) + const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates + /** + * `listingCapped` blocks the sync engine's deletion reconciliation, so it is set only + * when `maxCandidates` made the listing knowingly incomplete — candidates dropped from + * this page, or pages left unread behind the cap. Never when the cap coincides with + * genuine exhaustion, and never for the intentional `createdAfter` scope filter. + */ + if (syncContext && (droppedInPage || (hitLimit && sourceHasMore))) { + syncContext.listingCapped = true + } + + const hasMore = !hitLimit && sourceHasMore return { documents, @@ -518,21 +605,47 @@ export const ashbyConnector: ConnectorConfig = { externalId: string ): Promise => { try { - if (!externalId) return null + /** + * These are API-shape faults, not absence: `candidate.info` answered + * `success: true` with an unusable payload. Returning `null` would read as + * documented absence, and on an `add` the engine's `Promise.allSettled` + * hydration treats a fulfilled `null` as neither success nor failure — no + * `docsFailed`, no `failedExternalIds`, no log — so the candidate would + * vanish silently. Ashby sets `contentDeferred`, so this path is live. + */ + if (!externalId) throw new Error('Ashby getDocument called without a candidate id') const infoData = await ashbyPost(accessToken, 'candidate.info', { id: externalId }) - if (!infoData.results) return null + if (!infoData.results) { + throw new Error(`Ashby candidate.info returned no results for candidate ${externalId}`) + } const candidate = mapCandidate(infoData.results) - if (!candidate.id) return null + if (!candidate.id) { + throw new Error(`Ashby candidate.info returned a candidate with no id for ${externalId}`) + } const notes = await fetchAllNotes(accessToken, candidate.id) const feedback: AshbyFeedbackSummary[] = [] const applicationIds = candidate.applicationIds.slice(0, MAX_APPLICATIONS_FOR_FEEDBACK) + if (candidate.applicationIds.length > applicationIds.length) { + logger.warn('Truncated Ashby feedback fetch to the per-candidate application cap', { + externalId, + applications: candidate.applicationIds.length, + fetched: applicationIds.length, + }) + } + + /** + * Sequential on purpose. The sync engine already hydrates SYNC_BATCH_SIZE + * candidates concurrently, so fanning these out would multiply that into + * a burst of up to `MAX_APPLICATIONS_FOR_FEEDBACK` × the batch size + * simultaneous Ashby requests. A per-application catch keeps one failing + * application from losing the rest of the candidate's feedback. + */ for (const applicationId of applicationIds) { try { - const applicationFeedback = await fetchFeedbackForApplication(accessToken, applicationId) - feedback.push(...applicationFeedback) + feedback.push(...(await fetchFeedbackForApplication(accessToken, applicationId))) } catch (error) { logger.warn('Failed to fetch Ashby feedback for application', { applicationId, @@ -555,11 +668,17 @@ export const ashbyConnector: ConnectorConfig = { metadata: candidateMetadata(candidate), } } catch (error) { + /** + * Ashby documents no not-found code for `candidate.info`, so a thrown error cannot + * be read as absence — it is an HTTP fault or a `success: false` envelope. Rethrow + * so the sync engine records a failed row instead of treating a candidate that + * still exists as an empty re-fetch and leaving it silently stale. + */ logger.warn('Failed to get Ashby candidate', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/azure-devops/azure-devops.ts b/apps/sim/connectors/azure-devops/azure-devops.ts index 7351709a90d..52f90827590 100644 --- a/apps/sim/connectors/azure-devops/azure-devops.ts +++ b/apps/sim/connectors/azure-devops/azure-devops.ts @@ -232,6 +232,7 @@ interface WikiV2 { name: string remoteUrl?: string type?: string + isDisabled?: boolean } interface GitRepository { @@ -244,6 +245,17 @@ interface GitRepository { size?: number } +/** + * Resolves the browsable base URL for a repository. `webUrl` is declared on + * GitRepository but absent from the documented sample responses, so it cannot be + * relied on; `remoteUrl` appears in every sample as + * `https://dev.azure.com/{org}/{project}/_git/{repo}`, which is the same web + * route, and serves as the fallback. + */ +function repoBaseUrl(repo: GitRepository | undefined): string | undefined { + return repo?.webUrl || repo?.remoteUrl +} + interface GitItem { objectId: string gitObjectType?: string @@ -687,7 +699,13 @@ async function listRepositories( retryOptions?: Parameters[2], syncContext?: Record ): Promise { - const url = `${ADO_BASE_URL}/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/_apis/git/repositories?api-version=${GIT_API_VERSION}` + /** + * `includeAllUrls=true` — "True to include all remote URLs. The default value + * is false." The docs do not say which of GitRepository's URL fields it gates, + * and the sample listing omits `webUrl`, so it is requested to maximise the + * chance of getting one; `repoBaseUrl` falls back to `remoteUrl` regardless. + */ + const url = `${ADO_BASE_URL}/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/_apis/git/repositories?includeAllUrls=true&api-version=${GIT_API_VERSION}` const response = await fetchWithRetry( url, { @@ -743,11 +761,30 @@ async function resolveRepositories( if (syncContext && !cached) syncContext.repositories = all const needle = repositoryFilter.toLowerCase() - return all.filter((repo) => { + const matched = all.filter((repo) => { if (repo.isDisabled) return false if (!needle) return true return repo.id.toLowerCase() === needle || (repo.name ?? '').toLowerCase() === needle }) + + /** + * A configured repository filter that no longer resolves (renamed, deleted, + * or newly disabled repo) would otherwise produce an empty file listing and + * let reconciliation hard-delete every previously synced file. That is a + * stale reference, not evidence the files are gone, so the listing is flagged + * incomplete. An unfiltered project that genuinely has no repositories is + * left unflagged so real deletions still reconcile. + */ + if (needle && matched.length === 0 && all.length > 0) { + if (syncContext) syncContext.listingCapped = true + logger.warn('Configured Azure DevOps repository filter matched no repository', { + organization, + project, + repositoryFilter, + }) + } + + return matched } /** @@ -933,7 +970,7 @@ async function resolveRepoFiles( entries.push({ repoId: repo.id, repoName: repo.name, - repoWebUrl: repo.webUrl, + repoWebUrl: repoBaseUrl(repo), branch, item, }) @@ -1003,15 +1040,17 @@ async function resolveFileBranch( branchOverride: string, syncContext?: Record ): Promise<{ branch: string; repo?: GitRepository }> { - if (branchOverride) { - const repos = (syncContext?.repositories as GitRepository[] | undefined) ?? [] - return { branch: branchOverride, repo: repos.find((r) => r.id === repoId) } - } + /** + * The repository record is needed even when the branch is overridden — it + * supplies the web URL and display name that keep the hydrated document's + * sourceUrl and `repository` tag identical to the listing stub's. + */ const repos = (syncContext?.repositories as GitRepository[] | undefined) ?? (await listRepositories(accessToken, organization, project)) if (syncContext && !syncContext.repositories) syncContext.repositories = repos const repo = repos.find((r) => r.id === repoId) + if (branchOverride) return { branch: branchOverride, repo } return { branch: stripRefsHeads(repo?.defaultBranch ?? ''), repo } } @@ -1040,9 +1079,15 @@ async function getFileDocument( branchOverride, syncContext ) + /** + * A branch that will not resolve is a lookup failure, not an absent file — the + * file only reached hydration because the listing already resolved its repo. + * Returning `null` reads as documented absence, and on an `add` the engine's + * `Promise.allSettled` hydration counts a fulfilled `null` as neither success + * nor failure, so the file would vanish with no `docsFailed` and no log. + */ if (!branch) { - logger.warn('Cannot resolve branch for Azure DevOps file', { externalId }) - return null + throw new Error(`Cannot resolve branch for Azure DevOps file ${externalId}`) } const metadataParams = new URLSearchParams({ @@ -1064,8 +1109,15 @@ async function getFileDocument( throw new Error(`Failed to fetch repository file metadata: ${metadataResponse.status}`) } - const item = (await metadataResponse.json()) as GitItem - if (!item.objectId) return null + /** + * Items - Get declares a bare `GitItem` response, but every documented sample + * returns a `{ count, value: [...] }` collection (the samples scope with + * `scopePath` rather than `path`). Accept both shapes rather than depending on + * which one the service picks for this request. + */ + const metadataBody = (await metadataResponse.json()) as GitItem | { value?: GitItem[] } | null + const item = metadataBody && 'objectId' in metadataBody ? metadataBody : metadataBody?.value?.[0] + if (!item?.objectId) return null if (item.contentMetadata?.isBinary) { logger.info('Skipping binary Azure DevOps file', { path }) return null @@ -1105,7 +1157,7 @@ async function getFileDocument( title: skippedTitle, content: '', mimeType: 'text/plain', - sourceUrl: buildFileSourceUrl(repo?.webUrl, branch, path), + sourceUrl: buildFileSourceUrl(repoBaseUrl(repo), branch, path), contentHash: buildFileContentHash(repoId, item.objectId), metadata: { kind: 'file', @@ -1135,7 +1187,7 @@ async function getFileDocument( content, contentDeferred: false, mimeType: 'text/plain', - sourceUrl: buildFileSourceUrl(repo?.webUrl, branch, path), + sourceUrl: buildFileSourceUrl(repoBaseUrl(repo), branch, path), contentHash: buildFileContentHash(repoId, item.objectId), metadata: { kind: 'file', @@ -1197,9 +1249,25 @@ async function listWikiPages( syncContext?: Record ): Promise { const allWikis = await resolveWikis(accessToken, organization, project, syncContext) - const wikis = allWikis.filter((w) => wikiMatchesFilter(w, wikiFilter)) + const wikis = allWikis.filter((w) => !w.isDisabled && wikiMatchesFilter(w, wikiFilter)) if (wikis.length === 0) { + /** + * A configured wiki filter that no longer resolves (renamed, deleted, or + * newly disabled wiki) would otherwise produce an empty listing and let + * reconciliation hard-delete every previously synced page. That is a stale + * reference, not evidence the pages are gone, so the listing is flagged + * incomplete. A project that genuinely has no wikis is left unflagged so + * real deletions still reconcile. + */ + if (wikiFilter && allWikis.length > 0) { + if (syncContext) syncContext.listingCapped = true + logger.warn('Configured Azure DevOps wiki filter matched no wiki', { + organization, + project, + wikiFilter, + }) + } return { documents: [], hasMore: false } } @@ -1237,12 +1305,27 @@ async function listWikiPages( }) if (!response.ok) { const errorText = await response.text().catch(() => '') - logger.error('Failed to list Azure DevOps wiki pages', { + logger.error('Failed to list Azure DevOps wiki pages; skipping wiki', { wikiId: wiki.id, status: response.status, error: errorText, }) - throw new Error(`Failed to list wiki pages: ${response.status}`) + /** + * One unreadable wiki must not abort the whole sync (which would also drop + * the work-item and repository-file phases). Skip to the next wiki instead. + * Anything other than a 404 means the pages still exist but could not be + * read on this run, so the listing is flagged incomplete to keep deletion + * reconciliation from purging them. A 404 means the wiki is genuinely gone. + */ + if (response.status !== 404 && syncContext) { + syncContext.listingCapped = true + } + const moreWikis = wikiIndex + 1 < wikis.length + return { + documents: [], + nextCursor: moreWikis ? `wiki|${wikiIndex + 1}|` : undefined, + hasMore: moreWikis, + } } const data = await response.json() @@ -1386,7 +1469,8 @@ export const azureDevopsConnector: ConnectorConfig = { const wikiFilter = readString(sourceConfig.wikiName) const filters = readWorkItemFilters(sourceConfig) const fileFilters = readFileFilters(sourceConfig) - const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0 + const parsedMaxItems = Number(sourceConfig.maxItems) + const maxItems = Number.isFinite(parsedMaxItems) && parsedMaxItems > 0 ? parsedMaxItems : 0 if (!organization || !project) { throw new Error('Organization and project are required') @@ -1505,21 +1589,19 @@ export const azureDevopsConnector: ConnectorConfig = { * deferred wiki pages. Unknown IDs return null defensively. */ if (externalId.startsWith(FILE_PREFIX)) { - try { - return await getFileDocument( - accessToken, - organization, - project, - externalId, - readString(sourceConfig.branch), - syncContext - ) - } catch (error) { - logger.warn(`Failed to fetch Azure DevOps file ${externalId}`, { - error: toError(error).message, - }) - return null - } + /** + * `getFileDocument` returns null only for a 404 or an unreadable blob. Anything + * it throws is transient and propagates, so the sync engine records a failed row + * rather than hard-deleting an already-indexed file on a blip. + */ + return getFileDocument( + accessToken, + organization, + project, + externalId, + readString(sourceConfig.branch), + syncContext + ) } const parsed = parseWikiExternalId(externalId) diff --git a/apps/sim/connectors/azure-devops/meta.ts b/apps/sim/connectors/azure-devops/meta.ts index 939c9cb91e1..0f57df242d4 100644 --- a/apps/sim/connectors/azure-devops/meta.ts +++ b/apps/sim/connectors/azure-devops/meta.ts @@ -6,13 +6,14 @@ export const azureDevopsConnectorMeta: ConnectorMeta = { name: 'Azure DevOps', description: 'Sync wiki pages, work items, and repository files from an Azure DevOps project into your knowledge base', - version: '1.1.0', + version: '1.1.1', icon: AzureIcon, auth: { mode: 'apiKey', label: 'Personal Access Token', - placeholder: 'Enter your Azure DevOps PAT (scopes: Wiki Read, Work Items Read, Code Read)', + placeholder: + 'Enter your Azure DevOps PAT (scopes: Project and Team Read, Wiki Read, Work Items Read, Code Read)', }, /** diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index 2d82a9a6f2b..aa5bcae9a82 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { boxConnectorMeta } from '@/connectors/box/meta' @@ -354,7 +354,15 @@ async function fetchExtractedText( method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, }) - if (!response.ok && response.status !== 202) return null + /** + * A 404 means the representation is genuinely gone; 202 means still generating. + * Anything else (429, 5xx) is transient and must surface as a failed hydration rather + * than being folded into "this file has no extractable text". + */ + if (response.status === 404) return null + if (!response.ok && response.status !== 202) { + throw new Error(`Failed to poll Box representation status: ${response.status}`) + } if (response.status === 202) { state = 'pending' @@ -439,6 +447,17 @@ export const boxConnector: ConnectorConfig = { for (let fetched = 0; fetched < FOLDER_PAGES_PER_CALL && position; fetched++) { const page = await listFolderPage(accessToken, position.folderId, position.marker) + /** + * Losing access to a *sub*folder is survivable, but losing access to the + * configured root means the whole listing is empty. Failing loudly beats + * reporting a successful sync that indexed nothing. + */ + if (!page && position.folderId === rootFolderId) { + throw new Error( + `Box denied access to folder ${rootFolderId}. Reconnect the Box account or choose another folder.` + ) + } + if (page) { for (const item of page.entries ?? []) { if (item.type === 'folder') { @@ -470,7 +489,8 @@ export const boxConnector: ConnectorConfig = { ? { queue, folderId: position.folderId, marker: position.marker } : null - const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0 + const parsedMaxFiles = Number(sourceConfig.maxFiles) + const maxFiles = Number.isFinite(parsedMaxFiles) && parsedMaxFiles > 0 ? parsedMaxFiles : 0 const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 const stubs = files.map((item) => stubOrSkipBySize(fileToStub(item), item.size, MAX_FILE_SIZE)) @@ -506,60 +526,58 @@ export const boxConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - const response = await fetchWithRetry( - `${BOX_API_BASE}/files/${encodeURIComponent(externalId)}?fields=${FILE_FIELDS},representations`, - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'x-rep-hints': '[extracted_text]', - }, - } - ) - - if (response.status === 404 || response.status === 403) return null - if (!response.ok) { - throw new Error(`Failed to get Box file metadata: ${response.status}`) + const response = await fetchWithRetry( + `${BOX_API_BASE}/files/${encodeURIComponent(externalId)}?fields=${FILE_FIELDS},representations`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'x-rep-hints': '[extracted_text]', + }, } + ) - const file = (await response.json()) as BoxFileWithRepresentations - if (file.trashed_at || (file.item_status && file.item_status !== 'active')) return null - if (!isSupportedFile({ ...file, type: file.type ?? 'file' })) return null + /** + * 404/403 are the only statuses that mean the file is genuinely unavailable. + * Every other failure (429, 5xx, network faults) propagates so the sync engine + * records a failed row and excludes the file from deletion reconciliation. + */ + if (response.status === 404 || response.status === 403) return null + if (!response.ok) { + throw new Error(`Failed to get Box file metadata: ${response.status}`) + } - const stub = fileToStub(file) - if (file.size && file.size > MAX_FILE_SIZE) { - return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) - } + const file = (await response.json()) as BoxFileWithRepresentations + if (file.trashed_at || (file.item_status && file.item_status !== 'active')) return null + if (!isSupportedFile({ ...file, type: file.type ?? 'file' })) return null - const extension = getExtension(file) - - let content: string | null - try { - if (PLAIN_TEXT_EXTENSIONS.has(extension)) { - content = await fetchPlainTextContent(accessToken, file.id, extension) - } else { - const entry = (file.representations?.entries ?? []).find( - (candidate) => candidate.representation === 'extracted_text' - ) - content = entry ? await fetchExtractedText(accessToken, entry) : null - } - } catch (error) { - if (error instanceof ConnectorFileTooLargeError) { - return markSkipped(stub, sizeLimitSkipReason(error.limitBytes)) - } - throw error - } + const stub = fileToStub(file) + if (file.size && file.size > MAX_FILE_SIZE) { + return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) + } - if (!content?.trim()) return null + const extension = getExtension(file) - return { ...stub, content, contentDeferred: false } + let content: string | null + try { + if (PLAIN_TEXT_EXTENSIONS.has(extension)) { + content = await fetchPlainTextContent(accessToken, file.id, extension) + } else { + const entry = (file.representations?.entries ?? []).find( + (candidate) => candidate.representation === 'extracted_text' + ) + content = entry ? await fetchExtractedText(accessToken, entry) : null + } } catch (error) { - logger.warn(`Failed to fetch Box document ${externalId}`, { - error: toError(error).message, - }) - return null + if (error instanceof ConnectorFileTooLargeError) { + return markSkipped(stub, sizeLimitSkipReason(error.limitBytes)) + } + throw error } + + if (!content?.trim()) return null + + return { ...stub, content, contentDeferred: false } }, validateConfig: async ( diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 25b3dfff5cb..985b635b56f 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -4,8 +4,10 @@ import { describe, expect, it } from 'vitest' import { escapeCql, + extractCursor, isCurrentContent, preserveConfluenceCallouts, + readIncludedLabels, } from '@/connectors/confluence/confluence' import { htmlToPlainText } from '@/connectors/utils' @@ -54,6 +56,69 @@ describe('isCurrentContent', () => { }) }) +describe('extractCursor', () => { + it.concurrent('reads the cursor from a v1 CQL search next link', () => { + // Exact shape documented for GET /wiki/rest/api/content/search. + expect( + extractCursor('/rest/api/content/search?cql=type=page&limit=25&cursor=raNDoMsTRiNg') + ).toBe('raNDoMsTRiNg') + }) + + it.concurrent('reads the cursor from a v2 relative next link', () => { + expect(extractCursor('/wiki/api/v2/spaces/123/pages?limit=250&cursor=abc123')).toBe('abc123') + }) + + it.concurrent('url-decodes an encoded cursor value', () => { + expect(extractCursor('/rest/api/content/search?cursor=a%2Bb%2Fc%3D')).toBe('a+b/c=') + }) + + it.concurrent('returns undefined when the source is exhausted (no next link)', () => { + expect(extractCursor(undefined)).toBeUndefined() + expect(extractCursor(null)).toBeUndefined() + expect(extractCursor('')).toBeUndefined() + }) + + it.concurrent('returns undefined for a next link carrying no cursor', () => { + expect(extractCursor('/rest/api/content/search?cql=type=page&limit=25')).toBeUndefined() + }) + + it.concurrent('returns undefined for a non-string next link', () => { + expect(extractCursor({ href: '/x?cursor=a' })).toBeUndefined() + expect(extractCursor(42)).toBeUndefined() + }) + + it.concurrent('handles an absolute next link', () => { + expect(extractCursor('https://api.atlassian.com/wiki/api/v2/pages?cursor=xyz')).toBe('xyz') + }) +}) + +describe('readIncludedLabels', () => { + it.concurrent('reads names from the include-labels wrapper', () => { + expect( + readIncludedLabels({ + labels: { + results: [ + { id: '1', name: 'engineering' }, + { id: '2', name: 'published' }, + ], + }, + }) + ).toEqual(['engineering', 'published']) + }) + + it.concurrent('returns an empty array when labels are absent or empty', () => { + expect(readIncludedLabels({})).toEqual([]) + expect(readIncludedLabels({ labels: {} })).toEqual([]) + expect(readIncludedLabels({ labels: { results: [] } })).toEqual([]) + }) + + it.concurrent('drops entries with a missing or empty name rather than emitting blanks', () => { + expect( + readIncludedLabels({ labels: { results: [{ id: '1' }, { name: '' }, { name: 'kept' }] } }) + ).toEqual(['kept']) + }) +}) + describe('preserveConfluenceCallouts', () => { it.concurrent('handles empty content', () => { expect(preserveConfluenceCallouts('')).toBe('') diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 470cdc8ab68..381254d79f1 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -190,67 +190,32 @@ function buildSpaceClause(spaceKeys: string[]): string { } /** - * Fetches labels for a batch of page IDs using the v2 labels endpoint. + * Reads the `labels` field returned by the v2 single-content GET when + * `include-labels=true` is set, which removes the need for a second round trip + * to `/{type}/{id}/labels`. That embedded list is capped at 50 labels (the + * dedicated endpoint defaulted to 25), so this widens rather than narrows what + * a page can report; labels beyond the cap are paginated behind + * `labels._links` and are deliberately not followed. */ -const LABEL_FETCH_CONCURRENCY = 5 - -async function fetchLabelsForPages( - cloudId: string, - accessToken: string, - pageIds: string[] -): Promise> { - const labelsByPageId = new Map() - - for (let i = 0; i < pageIds.length; i += LABEL_FETCH_CONCURRENCY) { - const batch = pageIds.slice(i, i + LABEL_FETCH_CONCURRENCY) - const results = await Promise.all( - batch.map(async (pageId) => { - try { - let data: Record | null = null - for (const contentType of ['pages', 'blogposts']) { - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${contentType}/${pageId}/labels` - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (response.ok) { - data = await response.json() - break - } - if (response.status !== 404) { - logger.warn(`Failed to fetch labels for ${contentType} ${pageId}`, { - status: response.status, - }) - } - } - - if (!data) { - return { pageId, labels: [] as string[] } - } - - const labels = ((data.results as Record[]) || []).map( - (label) => label.name as string - ) - return { pageId, labels } - } catch (error) { - logger.warn(`Error fetching labels for page ${pageId}`, { - error: toError(error).message, - }) - return { pageId, labels: [] as string[] } - } - }) - ) +export function readIncludedLabels(page: Record): string[] { + const wrapper = page.labels as Record | undefined + const results = (wrapper?.results as Record[] | undefined) ?? [] + return results.map((label) => String(label.name ?? '')).filter(Boolean) +} - for (const { pageId, labels } of results) { - labelsByPageId.set(pageId, labels) - } +/** + * Extracts the `cursor` query value from a relative `_links.next` URL. Both the + * v2 endpoints and the v1 CQL search return the next page as a relative path + * carrying an opaque cursor, so the value has to be parsed back out rather than + * derived. + */ +export function extractCursor(nextLink: unknown): string | undefined { + if (typeof nextLink !== 'string' || !nextLink) return undefined + try { + return new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined + } catch { + return undefined } - - return labelsByPageId } /** @@ -418,7 +383,7 @@ export const confluenceConnector: ConnectorConfig = { */ let page: Record | null = null for (const endpoint of ['pages', 'blogposts']) { - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${externalId}?body-format=view` + const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${encodeURIComponent(externalId)}?body-format=view&include-labels=true` const response = await fetchWithRetry(url, { method: 'GET', headers: { @@ -442,13 +407,10 @@ export const confluenceConnector: ConnectorConfig = { const rawContent = (view?.value as string) || '' const plainText = htmlToPlainText(preserveConfluenceCallouts(rawContent)) - const labelMap = await fetchLabelsForPages(cloudId, accessToken, [String(page.id)]) - const labels = labelMap.get(String(page.id)) ?? [] - const links = page._links as Record | undefined const stub = pageToStub(page, { spaceId: page.spaceId, - labels, + labels: readIncludedLabels(page), sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined, }) @@ -590,20 +552,18 @@ async function listDocumentsV2( }) }) - let nextCursor: string | undefined - const nextLink = (data._links as Record)?.next - if (nextLink) { - try { - nextCursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined - } catch { - // Ignore malformed URLs - } - } + const nextCursor = extractCursor((data._links as Record | undefined)?.next) const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxPages > 0 && totalFetched >= maxPages - if (hitLimit && syncContext) syncContext.listingCapped = true + /** + * Only a cap that actually truncates a listing may suppress deletion + * reconciliation. When the source is exhausted (no next cursor) the listing is + * complete even though the count reached `maxPages`, and flagging it would + * permanently strand documents deleted upstream. + */ + if (hitLimit && nextCursor && syncContext) syncContext.listingCapped = true return { documents, @@ -697,6 +657,13 @@ async function listAllContentTypes( return results } +/** + * Page size for CQL search. The endpoint defaults to 25 and documents no hard + * maximum, so this stays conservatively below the fixed system limits it warns + * about rather than mirroring the v2 endpoints' 250. + */ +const CQL_PAGE_SIZE = 50 + /** * Lists documents using CQL search via the v1 API (used when label filtering is enabled). */ @@ -721,10 +688,17 @@ async function listDocumentsViaCql( if (contentType === 'blogpost') { cql += ' AND type="blogpost"' - } else if (contentType === 'page' || !contentType) { + } else if (contentType === 'all') { + /** + * An unconstrained CQL search matches every content type the index holds — + * attachments, comments, space descriptions and user profiles included — none + * of which `getDocument` can resolve through the page/blogpost endpoints. "All + * content" means both indexable content types, not literally everything. + */ + cql += ' AND type in ("page","blogpost")' + } else { cql += ' AND type="page"' } - // contentType === 'all' — no type filter if (labels.length === 1) { cql += ` AND label="${escapeCql(labels[0])}"` @@ -733,18 +707,33 @@ async function listDocumentsViaCql( cql += ` AND label in (${labelList})` } - const limit = maxPages > 0 ? Math.min(maxPages, 50) : 50 - const start = cursor ? Number(cursor) : 0 + const fetchedSoFar = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = maxPages > 0 ? maxPages - fetchedSoFar : Number.POSITIVE_INFINITY + /** + * The page size stays constant for every request of a run. This endpoint + * paginates by opaque cursor, and Atlassian does not document that a cursor + * issued against one `limit` stays valid when the following request asks for a + * different one, so narrowing `limit` to the remaining budget risks skipping or + * repeating results. The cap is applied by trimming the returned page instead. + */ const queryParams = new URLSearchParams() queryParams.append('cql', cql) - queryParams.append('limit', String(limit)) - queryParams.append('start', String(start)) + queryParams.append('limit', String(CQL_PAGE_SIZE)) queryParams.append('expand', 'version,metadata.labels') + /** + * `/wiki/rest/api/content/search` paginates by opaque cursor only — it has no + * `start` parameter, and its response carries no total count. The next page is + * reachable solely through the cursor embedded in `_links.next`. + */ + if (cursor) queryParams.append('cursor', cursor) const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/search?${queryParams.toString()}` - logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit }) + logger.info(`Searching Confluence via CQL: ${cql}`, { + limit: CQL_PAGE_SIZE, + hasCursor: Boolean(cursor), + }) const response = await fetchWithRetry(url, { method: 'GET', @@ -766,22 +755,36 @@ async function listDocumentsViaCql( const data = await response.json() const results = data.results || [] - const documents: ExternalDocument[] = (results as Record[]) + const allDocuments: ExternalDocument[] = (results as Record[]) .filter(isCurrentContent) .map((item) => cqlResultToStub(item, domain)) - const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length + /** + * Trim to the remaining budget. Trimming stops the walk (`hitLimit` below is + * then true), so the discarded tail is never skipped over — the run simply + * ends here. + */ + const documents = + allDocuments.length > remaining ? allDocuments.slice(0, remaining) : allDocuments + const trimmedByCap = documents.length < allDocuments.length + + const nextCursor = extractCursor((data._links as Record | undefined)?.next) + + const totalFetched = fetchedSoFar + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxPages > 0 && totalFetched >= maxPages - if (hitLimit && syncContext) syncContext.listingCapped = true + /** + * Both truncation shapes count: pages this run trimmed off, and a page left + * unread behind a live cursor. A cap that lands exactly on source exhaustion + * is a complete listing and must still reconcile deletions. + */ + if (hitLimit && (trimmedByCap || nextCursor) && syncContext) syncContext.listingCapped = true - const totalSize = (data.totalSize as number) ?? 0 - const nextStart = start + results.length - const hasMore = !hitLimit && nextStart < totalSize + const hasMore = !hitLimit && Boolean(nextCursor) return { documents, - nextCursor: hasMore ? String(nextStart) : undefined, + nextCursor: hasMore ? nextCursor : undefined, hasMore, } } diff --git a/apps/sim/connectors/discord/discord.ts b/apps/sim/connectors/discord/discord.ts index 412d8d28203..27ed685fafd 100644 --- a/apps/sim/connectors/discord/discord.ts +++ b/apps/sim/connectors/discord/discord.ts @@ -8,7 +8,29 @@ import { computeContentHash, parseTagDate } from '@/connectors/utils' const logger = createLogger('DiscordConnector') const DISCORD_API_BASE = 'https://discord.com/api/v10' +/** Discord caps `GET /channels/{id}/messages` at `limit=100`. */ const MESSAGES_PER_PAGE = 100 +/** Upper bound on `maxMessages` so a mistyped value cannot page the API forever. */ +const MAX_MESSAGES_CEILING = 50_000 +/** Discord snowflakes are numeric strings; reject anything else before path interpolation. */ +const SNOWFLAKE_PATTERN = /^\d{15,25}$/ + +/** + * Message types whose `content` is user-authored prose worth indexing. + * `0` = DEFAULT, `19` = REPLY. + */ +const INDEXABLE_MESSAGE_TYPES = new Set([0, 19]) + +/** + * Normalizes the optional `maxMessages` config value to a positive integer, + * falling back to the default for missing, non-numeric, or out-of-range input. + */ +function resolveMaxMessages(raw: unknown): number { + if (raw === undefined || raw === null || raw === '') return DEFAULT_MAX_MESSAGES + const parsed = Math.floor(Number(raw)) + if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_MAX_MESSAGES + return Math.min(parsed, MAX_MESSAGES_CEILING) +} interface DiscordMessage { id: string @@ -33,6 +55,30 @@ interface DiscordChannel { type: number } +/** + * Trims a user-supplied channel ID and returns it only if it is a Discord + * snowflake, so nothing else can be interpolated into an API path. + */ +function normalizeChannelId(raw: unknown): string | null { + if (typeof raw !== 'string') return null + const trimmed = raw.trim() + return SNOWFLAKE_PATTERN.test(trimmed) ? trimmed : null +} + +/** + * Carries the HTTP status so callers can tell a deleted channel (404 Unknown Channel) + * from a transient fault (429, 5xx) that must not be swallowed. + */ +class DiscordApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'DiscordApiError' + } +} + /** * Calls the Discord REST API with Bot token auth. * Unlike Slack, Discord returns proper HTTP status codes for errors. @@ -60,24 +106,30 @@ async function discordApiGet( if (!response.ok) { const body = await response.text().catch(() => '') - throw new Error(`Discord API error ${response.status}: ${body}`) + throw new DiscordApiError(`Discord API error ${response.status}: ${body}`, response.status) } return response.json() } /** - * Fetches all messages from a channel, up to a maximum count, using `before`-based pagination. + * Fetches all messages from a channel, up to a maximum count, using `before`-based + * snowflake pagination (Discord exposes no cursor and no total count). * Discord returns messages newest-first; we collect them all then reverse for chronological order. + * + * `truncated` is true when the `maxMessages` cap — rather than a short page — ended + * paging, so callers can log that older history may not be indexed. Discord reports no + * total count, so a channel holding exactly `maxMessages` messages also reports true. */ async function fetchChannelMessages( botToken: string, channelId: string, maxMessages: number -): Promise<{ messages: DiscordMessage[]; lastActivityTs?: string }> { +): Promise<{ messages: DiscordMessage[]; lastActivityTs?: string; truncated: boolean }> { const allMessages: DiscordMessage[] = [] let beforeId: string | undefined let lastActivityTs: string | undefined + let truncated = false while (allMessages.length < maxMessages) { const limit = Math.min(MESSAGES_PER_PAGE, maxMessages - allMessages.length) @@ -92,9 +144,9 @@ async function fetchChannelMessages( params )) as DiscordMessage[] - if (!messages || messages.length === 0) break + if (!Array.isArray(messages) || messages.length === 0) break - if (!lastActivityTs && messages.length > 0) { + if (!lastActivityTs) { lastActivityTs = messages[0].timestamp } @@ -103,34 +155,48 @@ async function fetchChannelMessages( // The last message in the batch is the oldest; use its ID for the next page beforeId = messages[messages.length - 1].id - // If we got fewer than requested, there are no more messages + // A short page means the channel history is exhausted; a full page means more remains if (messages.length < limit) break + if (allMessages.length >= maxMessages) truncated = true } - return { messages: allMessages.slice(0, maxMessages), lastActivityTs } + return { messages: allMessages.slice(0, maxMessages), lastActivityTs, truncated } } /** * Converts fetched messages into a single document content string. * Each line: "[ISO timestamp] username: message content" * Messages are returned chronologically (oldest first). + * + * `contentStripped` is true when indexable messages existed but every one of + * them had an empty `content`. Discord blanks `content` (along with `embeds`, + * `attachments`, and `components`) for applications without the + * `MESSAGE_CONTENT` privileged intent, so this shape almost always means the + * intent is disabled rather than that the channel is genuinely empty. */ -function formatMessages(messages: DiscordMessage[]): string { +function formatMessages(messages: DiscordMessage[]): { + content: string + contentStripped: boolean +} { const lines: string[] = [] + let indexableCount = 0 // Discord returns newest first; reverse for chronological order const chronological = [...messages].reverse() for (const msg of chronological) { - // Skip system messages (type 0 = DEFAULT, type 19 = REPLY are user messages) - if (msg.type !== 0 && msg.type !== 19) continue + if (!INDEXABLE_MESSAGE_TYPES.has(msg.type)) continue + indexableCount++ if (!msg.content) continue const userName = msg.author.username lines.push(`[${msg.timestamp}] ${userName}: ${msg.content}`) } - return lines.join('\n') + return { + content: lines.join('\n'), + contentStripped: indexableCount > 0 && lines.length === 0, + } } export const discordConnector: ConnectorConfig = { @@ -140,33 +206,51 @@ export const discordConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { - const channelId = sourceConfig.channelId as string - if (!channelId?.trim()) { - throw new Error('Channel ID is required') + const channelId = normalizeChannelId(sourceConfig.channelId) + if (!channelId) { + throw new Error('A valid numeric Discord channel ID is required') } - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) logger.info('Syncing Discord channel', { channelId, maxMessages }) - const channel = (await discordApiGet( - `/channels/${channelId.trim()}`, - accessToken - )) as DiscordChannel + const channel = (await discordApiGet(`/channels/${channelId}`, accessToken)) as DiscordChannel - const { messages, lastActivityTs } = await fetchChannelMessages( + const { messages, lastActivityTs, truncated } = await fetchChannelMessages( accessToken, - channel.id, + channelId, maxMessages ) - const content = formatMessages(messages) + if (truncated) { + logger.warn('Discord channel history truncated by maxMessages; older messages not indexed', { + channelId, + maxMessages, + }) + } + + const { content, contentStripped } = formatMessages(messages) if (!content.trim()) { - logger.info('No messages found in Discord channel', { channelId: channel.id }) + if (contentStripped) { + /** + * The channel still has messages, but Discord returned them with blank + * `content` — the `MESSAGE_CONTENT` privileged intent is almost certainly + * disabled for this bot. Emitting an empty listing here would let the sync + * engine hard-delete the already-stored channel document (the connector owns + * a single document, so it sits below the engine's suspect-empty-listing + * threshold). Cap the listing so deletion reconciliation is skipped. + */ + logger.warn( + 'Discord returned messages with empty content; enable the MESSAGE_CONTENT privileged intent for this bot', + { channelId, messageCount: messages.length } + ) + if (syncContext) syncContext.listingCapped = true + } else { + logger.info('No messages found in Discord channel', { channelId }) + } return { documents: [], hasMore: false } } @@ -200,23 +284,24 @@ export const discordConnector: ConnectorConfig = { sourceConfig: Record, externalId: string ): Promise => { - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) + + const channelId = normalizeChannelId(externalId) + if (!channelId) { + logger.warn('Discord getDocument called with a non-snowflake external ID', { externalId }) + return null + } try { - const channel = (await discordApiGet( - `/channels/${externalId}`, - accessToken - )) as DiscordChannel + const channel = (await discordApiGet(`/channels/${channelId}`, accessToken)) as DiscordChannel const { messages, lastActivityTs } = await fetchChannelMessages( accessToken, - externalId, + channelId, maxMessages ) - const content = formatMessages(messages) + const { content } = formatMessages(messages) if (!content.trim()) return null const contentHash = await computeContentHash(content) @@ -238,11 +323,20 @@ export const discordConnector: ConnectorConfig = { }, } } catch (error) { + /** + * Only a 404 (Unknown Channel) means the channel is genuinely gone. Every other + * failure — 429, 5xx, network faults — is rethrown so the sync engine records a + * failed row instead of silently dropping a channel that still exists. + */ + if (error instanceof DiscordApiError && error.status === 404) { + logger.info('Discord channel not found', { externalId }) + return null + } logger.warn('Failed to get Discord channel document', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, @@ -250,29 +344,35 @@ export const discordConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const channelId = sourceConfig.channelId as string | undefined + const rawChannelId = sourceConfig.channelId const maxMessages = sourceConfig.maxMessages as string | undefined - if (!channelId?.trim()) { + if (typeof rawChannelId !== 'string' || !rawChannelId.trim()) { return { valid: false, error: 'Channel ID is required' } } + const channelId = normalizeChannelId(rawChannelId) + if (!channelId) { + return { + valid: false, + error: 'Channel ID must be a Discord snowflake (a numeric ID, e.g. 123456789012345678)', + } + } + if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) { return { valid: false, error: 'Max messages must be a positive number' } } try { - await discordApiGet( - `/channels/${channelId.trim()}`, - accessToken, - undefined, - VALIDATE_RETRY_OPTIONS - ) + await discordApiGet(`/channels/${channelId}`, accessToken, undefined, VALIDATE_RETRY_OPTIONS) return { valid: true } } catch (error) { const message = getErrorMessage(error, 'Failed to validate configuration') if (message.includes('401') || message.includes('403')) { - return { valid: false, error: 'Invalid bot token or missing permissions for this channel' } + return { + valid: false, + error: 'Invalid bot token, or the bot lacks View Channel access to this channel', + } } if (message.includes('404')) { return { valid: false, error: `Channel not found: ${channelId}` } diff --git a/apps/sim/connectors/discord/meta.ts b/apps/sim/connectors/discord/meta.ts index 0793fb7dea6..8c7cb46236e 100644 --- a/apps/sim/connectors/discord/meta.ts +++ b/apps/sim/connectors/discord/meta.ts @@ -23,7 +23,8 @@ export const discordConnectorMeta: ConnectorMeta = { type: 'short-input', placeholder: 'e.g. 123456789012345678', required: true, - description: 'The Discord channel ID to sync messages from', + description: + 'The Discord channel ID to sync messages from. The bot must be in the server with View Channel and Read Message History permissions, and the MESSAGE_CONTENT privileged intent must be enabled — without it Discord returns every message with empty content.', }, { id: 'maxMessages', @@ -31,6 +32,7 @@ export const discordConnectorMeta: ConnectorMeta = { type: 'short-input', required: false, placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`, + description: 'Newest messages to index, counted back from the most recent.', }, ], diff --git a/apps/sim/connectors/docusign/docusign.ts b/apps/sim/connectors/docusign/docusign.ts index 66ea7a0b534..309eb3403f6 100644 --- a/apps/sim/connectors/docusign/docusign.ts +++ b/apps/sim/connectors/docusign/docusign.ts @@ -1,28 +1,13 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { getDocusignOAuthUrl, getDocusignWebBase } from '@/lib/oauth/docusign' import { docusignConnectorMeta } from '@/connectors/docusign/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' const logger = createLogger('DocuSignConnector') -/** - * DocuSign OAuth userinfo endpoint. Sim's DocuSign OAuth integration is wired to the - * demo/sandbox authorization server (`account-d.docusign.com`, see lib/oauth/oauth.ts token - * endpoint), so the connector resolves account info from the matching demo userinfo host. - * The production host is `https://account.docusign.com/oauth/userinfo`. - */ -const DOCUSIGN_USERINFO_URL = 'https://account-d.docusign.com/oauth/userinfo' - -/** - * DocuSign web-app base for envelope deep links. MUST match the same environment as - * {@link DOCUSIGN_USERINFO_URL}: demo/sandbox envelopes only exist in the demo web app - * (`appdemo.docusign.com`), not production (`app.docusign.com`). Keep these in lockstep - * if the OAuth environment ever changes. - */ -const DOCUSIGN_WEB_BASE = 'https://appdemo.docusign.com' - const DEFAULT_LOOKBACK_DAYS = 90 const MAX_PAGE_SIZE = 100 const DEFAULT_MAX_ENVELOPES = 0 @@ -167,7 +152,7 @@ async function resolveAccount( if (cached) return cached const response = await fetchWithRetry( - DOCUSIGN_USERINFO_URL, + getDocusignOAuthUrl('/oauth/userinfo'), { method: 'GET', headers: { @@ -227,7 +212,7 @@ function buildContentHash(envelope: DocuSignEnvelope): string { */ function buildSourceUrl(envelopeId: string | undefined): string | undefined { if (!envelopeId) return undefined - return `${DOCUSIGN_WEB_BASE}/documents/details/${envelopeId}` + return `${getDocusignWebBase()}/documents/details/${envelopeId}` } /** @@ -389,14 +374,27 @@ async function fetchFormValues( envelopeId: string ): Promise { try { - const response = await fetchWithRetry(`${apiBase}/envelopes/${envelopeId}/form_data`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - if (!response.ok) return [] + const response = await fetchWithRetry( + `${apiBase}/envelopes/${encodeURIComponent(envelopeId)}/form_data`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + } + ) + /** + * Only a 404 means the envelope carries no form data. Swallowing every other + * status would bake a permanently incomplete document: `buildContentHash` is + * metadata-only, so the next sync computes the identical hash, classifies the + * document `unchanged`, and the missing form-data section is never recovered + * until the envelope's status changes again. + */ + if (response.status === 404) return [] + if (!response.ok) { + throw new Error(`Failed to fetch DocuSign form data: ${response.status}`) + } const data = (await response.json()) as DocuSignFormData const values: DocuSignFormValue[] = [] if (Array.isArray(data.formData)) values.push(...data.formData) @@ -445,10 +443,21 @@ export const docusignConnector: ConnectorConfig = { : new Date(Date.now() - lookbackDays * MS_PER_DAY) if (syncContext && !cachedFromDate) syncContext.docusignFromDate = fromDate.toISOString() + /** + * Remaining budget under `maxEnvelopes`. The last page requests only what is still + * needed instead of a full {@link MAX_PAGE_SIZE} page. Safe to shrink because + * `start_position` is an offset the next page resumes from, not a page number. + * + * Floored to a positive integer: `maxEnvelopes` is free-form user input that + * `validateConfig` only checks for sign, and DocuSign rejects a fractional `count`. + */ + const remaining = maxEnvelopes > 0 ? maxEnvelopes - prevFetched : Number.POSITIVE_INFINITY + const pageSize = Math.max(1, Math.min(MAX_PAGE_SIZE, Math.floor(remaining))) + const queryParams = new URLSearchParams({ from_date: formatFromDate(fromDate), - include: 'recipients,custom_fields', - count: String(MAX_PAGE_SIZE), + include: 'recipients', + count: String(pageSize), start_position: String(startPosition), }) const statusFilter = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : '' @@ -480,35 +489,54 @@ export const docusignConnector: ConnectorConfig = { } const data = (await response.json()) as DocuSignEnvelopesListResponse - const envelopes = (data.envelopes ?? []).filter((e) => e.envelopeId) - const pageDocuments = envelopes.map(envelopeToStub) - - let documents = pageDocuments - if (maxEnvelopes > 0) { - const remaining = Math.max(0, maxEnvelopes - prevFetched) - if (pageDocuments.length > remaining) { - documents = pageDocuments.slice(0, remaining) - } + const rawEnvelopes = data.envelopes ?? [] + const envelopes = rawEnvelopes.filter((e) => e.envelopeId) + /** + * An entry the API returned without an `envelopeId` is dropped from the listing but + * still exists at the source, so the listing is no longer a complete source set. + */ + let capped = envelopes.length < rawEnvelopes.length + + let documents = envelopes.map(envelopeToStub) + if (documents.length > remaining) { + documents = documents.slice(0, remaining) + capped = true } const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitLimit = maxEnvelopes > 0 && totalFetched >= maxEnvelopes - if (hitLimit && syncContext) syncContext.listingCapped = true - + /** + * DocuSign reports paging position rather than a cursor, so `endPosition + 1 < + * totalSetSize` is the authoritative "more remains" signal — deliberately not page + * fullness, which under-reports whenever a page comes back short of `count`. The + * position-advance and non-empty checks only guard against a malformed page pinning + * `start_position` and looping forever. + */ const endPosition = Number(data.endPosition) const totalSetSize = Number(data.totalSetSize) - const hasNextPage = - pageDocuments.length === MAX_PAGE_SIZE && + const sourceHasMore = + rawEnvelopes.length > 0 && Number.isFinite(endPosition) && Number.isFinite(totalSetSize) && + endPosition + 1 > startPosition && endPosition + 1 < totalSetSize + /** + * `listingCapped` blocks the sync engine's deletion reconciliation. It is set only when + * the listing is genuinely incomplete — a `maxEnvelopes` cap reached while the source + * still has more envelopes, or a dropped entry — never on genuine exhaustion and never + * for the intentional `from_date` / `status` scope filters. + */ + const hitLimit = maxEnvelopes > 0 && totalFetched >= maxEnvelopes + if (syncContext && (capped || (hitLimit && sourceHasMore))) { + syncContext.listingCapped = true + } + return { documents, - nextCursor: !hitLimit && hasNextPage ? String(endPosition + 1) : undefined, - hasMore: !hitLimit && hasNextPage, + nextCursor: !hitLimit && sourceHasMore ? String(endPosition + 1) : undefined, + hasMore: !hitLimit && sourceHasMore, } }, @@ -525,7 +553,7 @@ export const docusignConnector: ConnectorConfig = { const apiBase = apiBaseFor(account) const response = await fetchWithRetry( - `${apiBase}/envelopes/${externalId}?include=recipients,custom_fields,documents`, + `${apiBase}/envelopes/${encodeURIComponent(externalId)}?include=recipients,custom_fields,documents`, { method: 'GET', headers: { @@ -566,11 +594,17 @@ export const docusignConnector: ConnectorConfig = { metadata: buildMetadata(envelope), } } catch (error) { + /** + * Documented absence — 404 and 410 — already returned `null` above. Anything + * reaching here (auth, 5xx, network) is a fault, and swallowing it would let the + * sync engine treat a still-existing envelope as an empty re-fetch and leave it + * silently stale instead of counting a failed document. + */ logger.warn('Failed to get DocuSign envelope', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index e4bad7b40d1..5e44cfb9edd 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { dropboxConnectorMeta } from '@/connectors/dropbox/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -34,52 +34,107 @@ const SUPPORTED_EXTENSIONS = new Set([ '.tsv', ]) +const HTML_EXTENSIONS = new Set(['.html', '.htm']) + const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES -interface DropboxFileEntry { - '.tag': 'file' | 'folder' | 'deleted' +/** Dropbox `FileMetadata` — the only `.tag` variant this connector indexes. */ +interface DropboxFileMetadata { + '.tag': 'file' id: string name: string path_lower: string path_display: string client_modified?: string server_modified?: string + rev?: string size?: number content_hash?: string is_downloadable?: boolean } +/** + * Dropbox serializes `Metadata` as a discriminated union. `folder` entries carry + * no size/modified fields and `deleted` entries carry no `id` at all, so they must + * be narrowed away before a stub is built from them. + */ +interface DropboxNonFileMetadata { + '.tag': 'folder' | 'deleted' + id?: string + name: string + path_lower?: string + path_display?: string +} + +type DropboxEntry = DropboxFileMetadata | DropboxNonFileMetadata + interface DropboxListFolderResponse { - entries: DropboxFileEntry[] + entries: DropboxEntry[] cursor: string has_more: boolean } -function hasSupportedExtension(name: string): boolean { +function extensionOf(name: string): string { const lower = name.toLowerCase() const dotIndex = lower.lastIndexOf('.') - if (dotIndex === -1) return false - return SUPPORTED_EXTENSIONS.has(lower.slice(dotIndex)) + return dotIndex === -1 ? '' : lower.slice(dotIndex) } /** A downloadable file with a supported extension, regardless of size. */ -function isDownloadableFile(entry: DropboxFileEntry): boolean { +function isDownloadableFile(entry: DropboxEntry): entry is DropboxFileMetadata { return ( - entry['.tag'] === 'file' && entry.is_downloadable !== false && hasSupportedExtension(entry.name) + entry['.tag'] === 'file' && + entry.is_downloadable !== false && + SUPPORTED_EXTENSIONS.has(extensionOf(entry.name)) ) } -async function downloadFileContent(accessToken: string, filePath: string): Promise { +/** + * Normalizes a user-supplied folder path to the `PathROrId` format + * `/2/files/list_folder` declares: the empty string for the Dropbox root (the + * leading-slash branch of the pattern is optional precisely so `""` matches), + * otherwise a leading slash. The trailing slash is stripped as defensive + * tidying of free-form input, not because Dropbox documents rejecting it. + * A path outside the format fails with `path/malformed_path`. + */ +function normalizeFolderPath(raw: unknown): string { + const trimmed = typeof raw === 'string' ? raw.trim() : '' + if (!trimmed || trimmed === '/') return '' + const withLeadingSlash = trimmed.startsWith('/') ? trimmed : `/${trimmed}` + return withLeadingSlash.replace(/\/+$/, '') +} + +/** + * Serializes the `Dropbox-API-Arg` header value as HTTP-header-safe ASCII. + * Dropbox requires DEL (0x7F) and every non-ASCII character to be sent as a JSON + * `\uXXXX` escape; a raw `JSON.stringify` of any argument carrying non-ASCII text + * fails with `could not decode input as JSON` or an invalid-header error. + * + * @see https://www.dropbox.com/developers/reference/json-encoding + */ +function toDropboxApiArg(arg: Record): string { + return JSON.stringify(arg).replace( + /[\u007f-\uffff]/g, + (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}` + ) +} + +async function downloadFileContent( + accessToken: string, + fileId: string, + isHtml: boolean +): Promise { const response = await fetchWithRetry('https://content.dropboxapi.com/2/files/download', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, - 'Dropbox-API-Arg': JSON.stringify({ path: filePath }), + 'Dropbox-API-Arg': toDropboxApiArg({ path: fileId }), }, }) if (!response.ok) { - throw new Error(`Failed to download file ${filePath}: ${response.status}`) + const errorText = await response.text().catch(() => '') + throw new Error(`Failed to download file ${fileId}: ${response.status} ${errorText}`.trim()) } // Stream with a hard byte cap so a file whose listing metadata under-reported @@ -92,22 +147,18 @@ async function downloadFileContent(accessToken: string, filePath: string): Promi const text = buffer.toString('utf8') - if (filePath.endsWith('.html') || filePath.endsWith('.htm')) { - return htmlToPlainText(text) - } - - return text + return isHtml ? htmlToPlainText(text) : text } -function fileToStub(entry: DropboxFileEntry): ExternalDocument { +function fileToStub(entry: DropboxFileMetadata): ExternalDocument { return { externalId: entry.id, title: entry.name, content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: `https://www.dropbox.com/home${entry.path_display}`, - contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified ?? ''}`, + sourceUrl: `https://www.dropbox.com/home${encodeURI(entry.path_display)}`, + contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.rev ?? entry.server_modified ?? ''}`, metadata: { path: entry.path_display, lastModified: entry.server_modified || entry.client_modified, @@ -151,8 +202,7 @@ export const dropboxConnector: ConnectorConfig = { data = await response.json() } else { - const folderPath = (sourceConfig.folderPath as string)?.trim() || '' - const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : '' + const path = normalizeFolderPath(sourceConfig.folderPath) logger.info('Listing Dropbox folder', { path: path || '(root)' }) @@ -187,7 +237,8 @@ export const dropboxConnector: ConnectorConfig = { // of dropping them silently at listing time. const candidateFiles = data.entries.filter(isDownloadableFile) - const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0 + const parsedMaxFiles = Number(sourceConfig.maxFiles) + const maxFiles = Number.isFinite(parsedMaxFiles) && parsedMaxFiles > 0 ? parsedMaxFiles : 0 const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 const stubs = candidateFiles.map((entry) => @@ -202,9 +253,19 @@ export const dropboxConnector: ConnectorConfig = { ) const totalFetched = previouslyFetched + indexableCount - if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = capReached - if (hitLimit && syncContext) syncContext.listingCapped = true + /** + * `listingCapped` blocks the sync engine's deletion reconciliation, so it is set + * only when the cap actually hid documents that still exist — either entries were + * dropped from this page, or Dropbox reported more pages we will not request. + * A cap that lands exactly on the last entry of the final page is a complete + * listing; flagging it would permanently block deletion reconciliation. + */ + const cappedWithItemsLeft = hitLimit && (documents.length < stubs.length || data.has_more) + if (syncContext) { + syncContext.totalDocsFetched = totalFetched + if (cappedWithItemsLeft) syncContext.listingCapped = true + } return { documents, @@ -218,48 +279,64 @@ export const dropboxConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/get_metadata', { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ path: externalId }), - }) - - if (!response.ok) { - if (response.status === 409) return null - throw new Error(`Failed to get metadata: ${response.status}`) + const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/get_metadata', { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: externalId }), + }) + + /** + * Dropbox reports every endpoint-specific error as 409, so the status alone + * does not mean the file is gone. For `get_metadata` the error union is + * `path: LookupError`, whose variants include `restricted_content` and + * `locked` — the file still exists in both. Only `not_found` is absence, and + * only that returns `null`; anything else propagates so the sync engine + * records a failed row instead of silently dropping the document. + */ + if (!response.ok) { + if (response.status === 409) { + const body = (await response.json().catch(() => null)) as { + error?: { '.tag'?: string; path?: { '.tag'?: string } } + } | null + if (body?.error?.['.tag'] === 'path' && body.error.path?.['.tag'] === 'not_found') { + return null + } } + throw new Error(`Failed to get metadata: ${response.status}`) + } - const entry = (await response.json()) as DropboxFileEntry - - if (!isDownloadableFile(entry)) return null + const entry = (await response.json()) as DropboxEntry - const stub = fileToStub(entry) - if (entry.size && entry.size > MAX_FILE_SIZE) { - return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) - } + if (!isDownloadableFile(entry)) return null - let content: string - try { - content = await downloadFileContent(accessToken, entry.path_lower) - } catch (error) { - if (error instanceof ConnectorFileTooLargeError) { - return markSkipped(stub, sizeLimitSkipReason(error.limitBytes)) - } - throw error - } - if (!content.trim()) return null + const stub = fileToStub(entry) + if (entry.size && entry.size > MAX_FILE_SIZE) { + return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) + } - return { ...stub, content, contentDeferred: false } + let content: string + try { + /** + * Addressed by file id rather than path: ids are stable across renames and + * are pure ASCII, so the `Dropbox-API-Arg` header stays header-safe. + */ + content = await downloadFileContent( + accessToken, + entry.id, + HTML_EXTENSIONS.has(extensionOf(entry.name)) + ) } catch (error) { - logger.warn(`Failed to fetch document ${externalId}`, { - error: toError(error).message, - }) - return null + if (error instanceof ConnectorFileTooLargeError) { + return markSkipped(stub, sizeLimitSkipReason(error.limitBytes)) + } + throw error } + if (!content.trim()) return null + + return { ...stub, content, contentDeferred: false } }, validateConfig: async ( @@ -272,8 +349,7 @@ export const dropboxConnector: ConnectorConfig = { } try { - const folderPath = (sourceConfig.folderPath as string)?.trim() || '' - const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : '' + const path = normalizeFolderPath(sourceConfig.folderPath) const response = await fetchWithRetry( 'https://api.dropboxapi.com/2/files/list_folder', diff --git a/apps/sim/connectors/evernote/evernote.ts b/apps/sim/connectors/evernote/evernote.ts deleted file mode 100644 index f8bc6711537..00000000000 --- a/apps/sim/connectors/evernote/evernote.ts +++ /dev/null @@ -1,545 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' -import { - ThriftReader, - ThriftWriter, - TYPE_I32, - TYPE_I64, - TYPE_LIST, - TYPE_STRING, - TYPE_STRUCT, -} from '@/app/api/tools/evernote/lib/thrift' -import { evernoteConnectorMeta } from '@/connectors/evernote/meta' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' - -const logger = createLogger('EvernoteConnector') - -const NOTES_PER_PAGE = 50 - -/** - * Extracts the shard ID from an Evernote developer token. - * Token format: "S=s1:U=12345:..." where s1 is the shard. - */ -function extractShardId(token: string): string { - const match = token.match(/S=s(\d+)/) - if (!match) { - throw new Error('Invalid Evernote token format: cannot extract shard ID') - } - return `s${match[1]}` -} - -/** - * Extracts the user ID from an Evernote developer token. - * Token format: "S=s1:U=12345:..." where 12345 is the user ID. - */ -function extractUserId(token: string): string { - const match = token.match(/:U=(\d+)/) - if (!match) { - throw new Error('Invalid Evernote token format: cannot extract user ID') - } - return match[1] -} - -/** - * Returns the Evernote API host based on the token type. - * Sandbox tokens contain `:Sandbox` and route to sandbox.evernote.com. - */ -function getHost(token: string): string { - return token.includes(':Sandbox') ? 'sandbox.evernote.com' : 'www.evernote.com' -} - -/** - * Derives the NoteStore URL from a developer token. - */ -function getNoteStoreUrl(token: string): string { - const shardId = extractShardId(token) - return `https://${getHost(token)}/shard/${shardId}/notestore` -} - -/** - * Sends a Thrift RPC call to the Evernote NoteStore via HTTP POST. - */ -async function callNoteStore( - token: string, - writer: ThriftWriter, - retryOptions?: Parameters[2] -): Promise { - const url = getNoteStoreUrl(token) - - const response = await fetchWithRetry( - url, - { - method: 'POST', - headers: { - 'Content-Type': 'application/x-thrift', - Accept: 'application/x-thrift', - }, - body: new Uint8Array(writer.toBuffer()), - }, - retryOptions - ) - - if (!response.ok) { - throw new Error(`Evernote HTTP ${response.status}: ${response.statusText}`) - } - - const reader = new ThriftReader(await response.arrayBuffer()) - const msg = reader.readMessageBegin() - - if (reader.isException(msg.type)) { - const ex = reader.readException() - throw new Error(`Evernote API error: ${ex.message}`) - } - - return reader -} - -/** - * Checks for Evernote-specific exceptions in response struct fields 1-3. - */ -function checkException(r: ThriftReader, fieldId: number, fieldType: number): boolean { - if ((fieldId === 1 || fieldId === 2) && fieldType === TYPE_STRUCT) { - let errorCode = 0 - let message = '' - r.readStruct((r2, fid, ftype) => { - if (fid === 1 && ftype === TYPE_I32) errorCode = r2.readI32() - else if (fid === 2 && ftype === TYPE_STRING) message = r2.readString() - else r2.skip(ftype) - }) - throw new Error(`Evernote error (${errorCode}): ${message}`) - } - if (fieldId === 3 && fieldType === TYPE_STRUCT) { - let identifier = '' - r.readStruct((r2, fid, ftype) => { - if (fid === 1 && ftype === TYPE_STRING) identifier = r2.readString() - else r2.skip(ftype) - }) - throw new Error(`Evernote not found: ${identifier}`) - } - return false -} - -interface Notebook { - guid: string - name: string -} - -interface Tag { - guid: string - name: string -} - -interface NoteMetadata { - guid: string - title: string - created: number - updated: number - notebookGuid: string - tagGuids: string[] -} - -interface Note { - guid: string - title: string - content: string - created: number - updated: number - notebookGuid: string - tagGuids: string[] -} - -async function apiListNotebooks( - token: string, - retryOptions?: Parameters[2] -): Promise { - const w = new ThriftWriter() - w.writeMessageBegin('listNotebooks', 0) - w.writeStringField(1, token) - w.writeFieldStop() - - const r = await callNoteStore(token, w, retryOptions) - const notebooks: Notebook[] = [] - - r.readStruct((r2, fid, ftype) => { - if (fid === 0 && ftype === TYPE_LIST) { - const { size } = r2.readListBegin() - for (let i = 0; i < size; i++) { - let guid = '' - let name = '' - r2.readStruct((r3, fid3, ftype3) => { - if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString() - else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString() - else r3.skip(ftype3) - }) - notebooks.push({ guid, name }) - } - } else if (!checkException(r2, fid, ftype)) { - r2.skip(ftype) - } - }) - - return notebooks -} - -async function apiListTags( - token: string, - retryOptions?: Parameters[2] -): Promise { - const w = new ThriftWriter() - w.writeMessageBegin('listTags', 0) - w.writeStringField(1, token) - w.writeFieldStop() - - const r = await callNoteStore(token, w, retryOptions) - const tags: Tag[] = [] - - r.readStruct((r2, fid, ftype) => { - if (fid === 0 && ftype === TYPE_LIST) { - const { size } = r2.readListBegin() - for (let i = 0; i < size; i++) { - let guid = '' - let name = '' - r2.readStruct((r3, fid3, ftype3) => { - if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString() - else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString() - else r3.skip(ftype3) - }) - tags.push({ guid, name }) - } - } else if (!checkException(r2, fid, ftype)) { - r2.skip(ftype) - } - }) - - return tags -} - -/** - * Calls NoteStore.findNotesMetadata with offset-based pagination. - * - * Thrift field numbers (from NoteStore.thrift): - * findNotesMetadata(1:token, 2:NoteFilter, 3:offset, 4:maxNotes, 5:ResultSpec) - * NoteFilter: 4:notebookGuid - * NotesMetadataResultSpec: 2:includeTitle, 6:includeCreated, 7:includeUpdated, - * 11:includeNotebookGuid, 12:includeTagGuids - * NotesMetadataList: 1:startIndex, 2:totalNotes, 3:list - * NoteMetadata: 1:guid, 2:title, 6:created(i64), 7:updated(i64), - * 11:notebookGuid, 12:list - */ -async function apiFindNotesMetadata( - token: string, - offset: number, - maxNotes: number, - notebookGuid?: string, - retryOptions?: Parameters[2] -): Promise<{ totalNotes: number; notes: NoteMetadata[] }> { - const w = new ThriftWriter() - w.writeMessageBegin('findNotesMetadata', 0) - w.writeStringField(1, token) - - w.writeFieldBegin(TYPE_STRUCT, 2) // NoteFilter - if (notebookGuid) { - w.writeStringField(4, notebookGuid) - } - w.writeFieldStop() - - w.writeI32Field(3, offset) - w.writeI32Field(4, maxNotes) - - w.writeFieldBegin(TYPE_STRUCT, 5) // NotesMetadataResultSpec - w.writeBoolField(2, true) // includeTitle - w.writeBoolField(6, true) // includeCreated - w.writeBoolField(7, true) // includeUpdated - w.writeBoolField(11, true) // includeNotebookGuid - w.writeBoolField(12, true) // includeTagGuids - w.writeFieldStop() - - w.writeFieldStop() - - const r = await callNoteStore(token, w, retryOptions) - - let totalNotes = 0 - const notes: NoteMetadata[] = [] - - r.readStruct((r2, fid, ftype) => { - if (fid === 0 && ftype === TYPE_STRUCT) { - r2.readStruct((r3, fid3, ftype3) => { - if (fid3 === 1 && ftype3 === TYPE_I32) { - r3.readI32() - } else if (fid3 === 2 && ftype3 === TYPE_I32) { - totalNotes = r3.readI32() - } else if (fid3 === 3 && ftype3 === TYPE_LIST) { - const { size } = r3.readListBegin() - for (let i = 0; i < size; i++) { - let guid = '' - let title = '' - let created = 0 - let updated = 0 - let nbGuid = '' - const tagGuids: string[] = [] - - r3.readStruct((r4, fid4, ftype4) => { - if (fid4 === 1 && ftype4 === TYPE_STRING) guid = r4.readString() - else if (fid4 === 2 && ftype4 === TYPE_STRING) title = r4.readString() - else if (fid4 === 6 && ftype4 === TYPE_I64) created = Number(r4.readI64()) - else if (fid4 === 7 && ftype4 === TYPE_I64) updated = Number(r4.readI64()) - else if (fid4 === 11 && ftype4 === TYPE_STRING) nbGuid = r4.readString() - else if (fid4 === 12 && ftype4 === TYPE_LIST) { - const { size: tagCount } = r4.readListBegin() - for (let t = 0; t < tagCount; t++) tagGuids.push(r4.readString()) - } else { - r4.skip(ftype4) - } - }) - notes.push({ guid, title, created, updated, notebookGuid: nbGuid, tagGuids }) - } - } else { - r3.skip(ftype3) - } - }) - } else if (!checkException(r2, fid, ftype)) { - r2.skip(ftype) - } - }) - - return { totalNotes, notes } -} - -/** - * Calls NoteStore.getNote to fetch a single note with content. - * - * Thrift: getNote(1:token, 2:guid, 3:withContent, 4:withResourcesData, - * 5:withResourcesRecognition, 6:withResourcesAlternateData) - * Note: 1:guid, 2:title, 3:content, 6:created, 7:updated, 11:notebookGuid, 12:tagGuids - */ -async function apiGetNote( - token: string, - guid: string, - retryOptions?: Parameters[2] -): Promise { - const w = new ThriftWriter() - w.writeMessageBegin('getNote', 0) - w.writeStringField(1, token) - w.writeStringField(2, guid) - w.writeBoolField(3, true) // withContent - w.writeBoolField(4, false) // withResourcesData - w.writeBoolField(5, false) // withResourcesRecognition - w.writeBoolField(6, false) // withResourcesAlternateData - w.writeFieldStop() - - const r = await callNoteStore(token, w, retryOptions) - - let noteGuid = '' - let title = '' - let content = '' - let created = 0 - let updated = 0 - let notebookGuid = '' - const tagGuids: string[] = [] - - r.readStruct((r2, fid, ftype) => { - if (fid === 0 && ftype === TYPE_STRUCT) { - r2.readStruct((r3, fid3, ftype3) => { - if (fid3 === 1 && ftype3 === TYPE_STRING) noteGuid = r3.readString() - else if (fid3 === 2 && ftype3 === TYPE_STRING) title = r3.readString() - else if (fid3 === 3 && ftype3 === TYPE_STRING) content = r3.readString() - else if (fid3 === 6 && ftype3 === TYPE_I64) created = Number(r3.readI64()) - else if (fid3 === 7 && ftype3 === TYPE_I64) updated = Number(r3.readI64()) - else if (fid3 === 11 && ftype3 === TYPE_STRING) notebookGuid = r3.readString() - else if (fid3 === 12 && ftype3 === TYPE_LIST) { - const { size } = r3.readListBegin() - for (let t = 0; t < size; t++) tagGuids.push(r3.readString()) - } else { - r3.skip(ftype3) - } - }) - } else if (!checkException(r2, fid, ftype)) { - r2.skip(ftype) - } - }) - - return { guid: noteGuid || guid, title, content, created, updated, notebookGuid, tagGuids } -} - -export const evernoteConnector: ConnectorConfig = { - ...evernoteConnectorMeta, - - listDocuments: async ( - accessToken: string, - sourceConfig: Record, - cursor?: string, - syncContext?: Record - ): Promise => { - const notebookGuid = (sourceConfig.notebookGuid as string) || undefined - const retryOptions = { maxRetries: 3, initialDelayMs: 500 } - - if (syncContext && !syncContext.tagMap) { - const tags = await apiListTags(accessToken, retryOptions) - syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name])) - } - if (syncContext && !syncContext.notebookMap) { - const notebooks = await apiListNotebooks(accessToken, retryOptions) - syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name])) - } - - const tagMap = (syncContext?.tagMap as Record) || {} - const notebookMap = (syncContext?.notebookMap as Record) || {} - const offset = cursor ? Number(cursor) : 0 - const shardId = extractShardId(accessToken) - const userId = extractUserId(accessToken) - const host = getHost(accessToken) - - logger.info('Listing Evernote notes', { offset, maxNotes: NOTES_PER_PAGE }) - - const result = await apiFindNotesMetadata( - accessToken, - offset, - NOTES_PER_PAGE, - notebookGuid, - retryOptions - ) - - const documents: ExternalDocument[] = result.notes.map((meta) => { - const tagNames = meta.tagGuids.map((g) => tagMap[g]).filter(Boolean) - - return { - externalId: meta.guid, - title: meta.title || 'Untitled', - content: '', - contentDeferred: true, - mimeType: 'text/plain', - sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${meta.guid}/`, - contentHash: `evernote:${meta.guid}:${meta.updated}`, - metadata: { - tags: tagNames, - notebook: notebookMap[meta.notebookGuid] || '', - createdAt: meta.created ? new Date(meta.created).toISOString() : undefined, - updatedAt: meta.updated ? new Date(meta.updated).toISOString() : undefined, - }, - } - }) - - const nextOffset = offset + result.notes.length - const hasMore = nextOffset < result.totalNotes - - return { - documents, - nextCursor: hasMore ? String(nextOffset) : undefined, - hasMore, - } - }, - - getDocument: async ( - accessToken: string, - _sourceConfig: Record, - externalId: string, - syncContext?: Record - ): Promise => { - try { - const retryOptions = { maxRetries: 3, initialDelayMs: 500 } - const note = await apiGetNote(accessToken, externalId, retryOptions) - const plainText = htmlToPlainText(note.content) - const title = note.title || 'Untitled' - const content = plainText.trim() ? plainText : title - - const shardId = extractShardId(accessToken) - const userId = extractUserId(accessToken) - const host = getHost(accessToken) - - if (syncContext && !syncContext.tagMap) { - const tags = await apiListTags(accessToken, retryOptions) - syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name])) - } - if (syncContext && !syncContext.notebookMap) { - const notebooks = await apiListNotebooks(accessToken, retryOptions) - syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name])) - } - - let tagMap: Record - let notebookMap: Record - if (syncContext) { - tagMap = syncContext.tagMap as Record - notebookMap = syncContext.notebookMap as Record - } else { - const tags = await apiListTags(accessToken, retryOptions) - tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name])) - const notebooks = await apiListNotebooks(accessToken, retryOptions) - notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name])) - } - - const tagNames = note.tagGuids.map((g) => tagMap[g]).filter(Boolean) - const notebookName = notebookMap[note.notebookGuid] || '' - - return { - externalId, - title, - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${externalId}/`, - contentHash: `evernote:${note.guid}:${note.updated}`, - metadata: { - tags: tagNames, - notebook: notebookName, - createdAt: note.created ? new Date(note.created).toISOString() : undefined, - updatedAt: note.updated ? new Date(note.updated).toISOString() : undefined, - }, - } - } catch (error) { - logger.warn('Failed to get Evernote note', { - externalId, - error: toError(error).message, - }) - return null - } - }, - - validateConfig: async ( - accessToken: string, - sourceConfig: Record - ): Promise<{ valid: boolean; error?: string }> => { - try { - extractShardId(accessToken) - } catch { - return { valid: false, error: 'Invalid developer token format — must start with S=s{number}' } - } - - try { - const notebooks = await apiListNotebooks(accessToken, VALIDATE_RETRY_OPTIONS) - - const notebookGuid = (sourceConfig.notebookGuid as string) || '' - if (notebookGuid.trim()) { - const found = notebooks.some((nb) => nb.guid === notebookGuid.trim()) - if (!found) { - return { valid: false, error: `Notebook with GUID "${notebookGuid}" not found` } - } - } - - return { valid: true } - } catch (error) { - const message = toError(error).message || 'Failed to connect to Evernote' - return { valid: false, error: message } - } - }, - - mapTags: (metadata: Record): Record => { - const result: Record = {} - - const tags = joinTagArray(metadata.tags) - if (tags) result.tags = tags - - if (typeof metadata.notebook === 'string' && metadata.notebook) { - result.notebook = metadata.notebook - } - - const updatedAt = parseTagDate(metadata.updatedAt) - if (updatedAt) result.updatedAt = updatedAt - - const createdAt = parseTagDate(metadata.createdAt) - if (createdAt) result.createdAt = createdAt - - return result - }, -} diff --git a/apps/sim/connectors/evernote/index.ts b/apps/sim/connectors/evernote/index.ts deleted file mode 100644 index 4bd08dfa2eb..00000000000 --- a/apps/sim/connectors/evernote/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { evernoteConnector } from '@/connectors/evernote/evernote' diff --git a/apps/sim/connectors/evernote/meta.ts b/apps/sim/connectors/evernote/meta.ts deleted file mode 100644 index d60f1e8b2a9..00000000000 --- a/apps/sim/connectors/evernote/meta.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { EvernoteIcon } from '@/components/icons' -import type { ConnectorMeta } from '@/connectors/types' - -export const evernoteConnectorMeta: ConnectorMeta = { - id: 'evernote', - name: 'Evernote', - description: 'Sync notes from Evernote', - version: '1.0.0', - icon: EvernoteIcon, - - auth: { - mode: 'apiKey', - label: 'Developer Token', - placeholder: 'Enter your Evernote developer token (starts with S=)', - }, - - configFields: [ - { - id: 'notebookGuid', - title: 'Notebook GUID', - type: 'short-input', - placeholder: 'Leave empty to sync all notebooks', - required: false, - description: 'Sync only notes from this notebook (optional)', - }, - ], - - tagDefinitions: [ - { id: 'tags', displayName: 'Tags', fieldType: 'text' }, - { id: 'notebook', displayName: 'Notebook', fieldType: 'text' }, - { id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' }, - { id: 'createdAt', displayName: 'Created', fieldType: 'date' }, - ], -} diff --git a/apps/sim/connectors/fathom/fathom.ts b/apps/sim/connectors/fathom/fathom.ts index 760766d2ff1..bce6b9f4ff3 100644 --- a/apps/sim/connectors/fathom/fathom.ts +++ b/apps/sim/connectors/fathom/fathom.ts @@ -90,19 +90,18 @@ interface FathomSummaryResponse { } /** - * Header fields cached per recording during `listDocuments` so `getDocument` - * can render an identical document header. Fathom exposes no single-meeting + * Everything about a meeting derivable from the listing response alone. Both the + * listing stub and the hydrated document are built from it, so their + * `contentHash` values are identical by construction. + * + * Cached per recording during `listDocuments`: Fathom exposes no single-meeting * GET and no `recording_ids` filter, so this metadata cannot be refetched once * listing has moved past the page that contained it — it is carried forward in * the shared `syncContext` instead. */ interface FathomMeetingHeader { + externalId: string title: string - meetingDate?: string - durationSeconds?: number - recordedByEmail?: string - recordedByName?: string - team?: string sourceUrl?: string contentHash: string metadata: FathomMeetingMetadata @@ -182,16 +181,14 @@ function buildMetadata(meeting: FathomMeeting): FathomMeetingMetadata { } /** - * Extracts the lightweight header fields cached for `getDocument`. + * Extracts the lightweight header fields cached for `getDocument`. Returns null + * for a meeting with no `recording_id` — nothing about it is addressable. */ -function buildHeader(meeting: FathomMeeting): FathomMeetingHeader { +function buildHeader(meeting: FathomMeeting): FathomMeetingHeader | null { + if (meeting.recording_id == null) return null return { + externalId: String(meeting.recording_id), title: resolveTitle(meeting), - meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined, - durationSeconds: computeDurationSeconds(meeting), - recordedByEmail: meeting.recorded_by?.email, - recordedByName: meeting.recorded_by?.name, - team: meeting.recorded_by?.team ?? undefined, sourceUrl: buildSourceUrl(meeting), contentHash: buildContentHash(meeting), metadata: buildMetadata(meeting), @@ -219,23 +216,30 @@ function readCachedHeader( syncContext: Record | undefined, recordingId: string ): FathomMeetingHeader | undefined { - const cache = syncContext?.meetingHeaders as Record | undefined - return cache?.[recordingId] + const cache = syncContext?.meetingHeaders as Map | undefined + return cache?.get(recordingId) } /** * Stores the header for a recording in the shared sync context. + * + * Deliberately uncapped: Fathom exposes no single-meeting GET and no + * `recording_ids` list filter, so a header evicted here could never be + * recovered and its meeting would fail to hydrate. The cache holds one small + * object per listed meeting — the same order of magnitude as the stub list the + * sync engine already retains for the whole run. */ function cacheHeader( syncContext: Record | undefined, - recordingId: string, header: FathomMeetingHeader ): void { if (!syncContext) return - const cache = - (syncContext.meetingHeaders as Record | undefined) ?? {} - cache[recordingId] = header - syncContext.meetingHeaders = cache + let cache = syncContext.meetingHeaders as Map | undefined + if (!cache) { + cache = new Map() + syncContext.meetingHeaders = cache + } + cache.set(header.externalId, header) } /** @@ -243,25 +247,24 @@ function cacheHeader( * plain-text document with one `Speaker: text` line per transcript entry. */ function formatMeetingContent( - header: FathomMeetingHeader | undefined, + header: FathomMeetingHeader, transcript: FathomTranscriptEntry[], summary: FathomSummary | null ): string { const parts: string[] = [] + const { meetingDate, durationSeconds, recordedByEmail, recordedByName, team } = header.metadata - parts.push(`Meeting: ${header?.title ?? 'Untitled Fathom Meeting'}`) + parts.push(`Meeting: ${header.title}`) - if (header?.meetingDate) parts.push(`Date: ${header.meetingDate}`) + if (meetingDate) parts.push(`Date: ${meetingDate}`) - if (header?.durationSeconds != null) { - parts.push(`Duration: ${Math.round(header.durationSeconds / 60)} minutes`) + if (durationSeconds != null) { + parts.push(`Duration: ${Math.round(durationSeconds / 60)} minutes`) } - if (header?.recordedByEmail) { - parts.push(`Recorded by: ${header.recordedByName ?? header.recordedByEmail}`) - } + if (recordedByEmail) parts.push(`Recorded by: ${recordedByName ?? recordedByEmail}`) - if (header?.team) parts.push(`Team: ${header.team}`) + if (team) parts.push(`Team: ${team}`) if (summary?.markdown_formatted?.trim()) { parts.push('') @@ -283,20 +286,20 @@ function formatMeetingContent( } /** - * Converts a listing meeting into a deferred stub. Content is fetched lazily - * via `getDocument` only for new or changed meetings. + * Converts a cached header into a deferred stub. Content is fetched lazily via + * `getDocument` only for new or changed meetings, and both documents are built + * from the same header so their `contentHash` is identical by construction. */ -function meetingToStub(meeting: FathomMeeting): ExternalDocument { - const metadata = buildMetadata(meeting) +function headerToStub(header: FathomMeetingHeader): ExternalDocument { return { - externalId: String(meeting.recording_id), - title: resolveTitle(meeting), + externalId: header.externalId, + title: header.title, content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: buildSourceUrl(meeting), - contentHash: buildContentHash(meeting), - metadata: { ...metadata }, + sourceUrl: header.sourceUrl, + contentHash: header.contentHash, + metadata: { ...header.metadata }, } } @@ -358,25 +361,37 @@ export const fathomConnector: ConnectorConfig = { const allDocuments: ExternalDocument[] = [] for (const meeting of meetings) { - if (meeting.recording_id == null) continue - const externalId = String(meeting.recording_id) - cacheHeader(syncContext, externalId, buildHeader(meeting)) - allDocuments.push(meetingToStub(meeting)) + const header = buildHeader(meeting) + if (!header) continue + cacheHeader(syncContext, header) + allDocuments.push(headerToStub(header)) } const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let documents = allDocuments + let capDroppedDocs = false if (maxMeetings > 0) { const remaining = Math.max(0, maxMeetings - prevFetched) if (allDocuments.length > remaining) { documents = allDocuments.slice(0, remaining) + capDroppedDocs = true } } const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxMeetings > 0 && totalFetched >= maxMeetings - if (hitLimit && syncContext) syncContext.listingCapped = true + + /** + * The listing is only incomplete when the cap actually hid meetings — either + * by dropping some from this page or by stopping while another cursor + * remains. Reaching the cap exactly at source exhaustion (nothing dropped, + * no further cursor) yields a complete listing, so deletion reconciliation + * must still run for meetings removed in Fathom. + */ + if (syncContext && (capDroppedDocs || (hitLimit && Boolean(nextCursor)))) { + syncContext.listingCapped = true + } const hasMore = !hitLimit && Boolean(nextCursor) @@ -387,85 +402,84 @@ export const fathomConnector: ConnectorConfig = { } }, + /** + * Hydrates a listing stub with its transcript and, when available, its summary. + * + * Returns `null` only when the meeting genuinely has nothing to index (recording + * gone, transcript and summary both still processing). Transport, rate-limit, and + * server errors propagate so the sync engine records them as failed documents + * instead of reporting a clean sync that silently dropped meetings. + */ getDocument: async ( accessToken: string, _sourceConfig: Record, externalId: string, syncContext?: Record ): Promise => { - try { - if (!externalId) return null - - const transcriptUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/transcript` - const transcriptResponse = await fetchWithRetry(transcriptUrl, { - method: 'GET', - headers: buildHeaders(accessToken), - }) - - if (!transcriptResponse.ok) { - if (transcriptResponse.status === 404) return null - throw new Error(`Failed to fetch Fathom transcript: ${transcriptResponse.status}`) - } - - const transcriptData = (await transcriptResponse.json()) as FathomTranscriptResponse - const transcript = transcriptData.transcript ?? [] - - let summary: FathomSummary | null = null - try { - const summaryUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/summary` - const summaryResponse = await fetchWithRetry(summaryUrl, { - method: 'GET', - headers: buildHeaders(accessToken), - }) - if (summaryResponse.ok) { - const summaryData = (await summaryResponse.json()) as FathomSummaryResponse - summary = summaryData.summary ?? null - } - } catch (summaryError) { - logger.warn('Failed to fetch Fathom summary', { - externalId, - error: toError(summaryError).message, - }) - } + if (!externalId) return null + + /** + * Resolved before any network call: without the listing header this meeting + * cannot be rendered, and Fathom offers no way to refetch it. + */ + const header = readCachedHeader(syncContext, externalId) + if (!header) { + throw new Error(`No cached Fathom listing header for recording ${externalId}`) + } - const hasTranscript = transcript.some((entry) => entry.text?.trim()) - const hasSummary = Boolean(summary?.markdown_formatted?.trim()) - if (!hasTranscript && !hasSummary) { - logger.info('No transcript or summary yet for Fathom meeting', { externalId }) - return null - } + const transcriptUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/transcript` + const transcriptResponse = await fetchWithRetry(transcriptUrl, { + method: 'GET', + headers: buildHeaders(accessToken), + }) - const header = readCachedHeader(syncContext, externalId) - if (!header) { - logger.warn( - 'No cached header for Fathom meeting; skipping to avoid an un-refreshable record', - { - externalId, - } - ) - return null - } + if (!transcriptResponse.ok) { + if (transcriptResponse.status === 404) return null + throw new Error(`Failed to fetch Fathom transcript: ${transcriptResponse.status}`) + } - const content = formatMeetingContent(header, transcript, summary).trim() - if (!content) return null + const transcriptData = (await transcriptResponse.json()) as FathomTranscriptResponse + const transcript = transcriptData.transcript ?? [] - return { - externalId, - title: header.title, - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: header.sourceUrl, - contentHash: header.contentHash, - metadata: { ...header.metadata }, + /** + * The summary is optional enrichment, so a failure to read it degrades the + * document rather than failing the meeting. + */ + let summary: FathomSummary | null = null + try { + const summaryUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/summary` + const summaryResponse = await fetchWithRetry(summaryUrl, { + method: 'GET', + headers: buildHeaders(accessToken), + }) + if (summaryResponse.ok) { + const summaryData = (await summaryResponse.json()) as FathomSummaryResponse + summary = summaryData.summary ?? null } - } catch (error) { - logger.warn('Failed to get Fathom meeting', { + } catch (summaryError) { + logger.warn('Failed to fetch Fathom summary', { externalId, - error: toError(error).message, + error: toError(summaryError).message, }) + } + + const hasTranscript = transcript.some((entry) => entry.text?.trim()) + const hasSummary = Boolean(summary?.markdown_formatted?.trim()) + if (!hasTranscript && !hasSummary) { + logger.info('No transcript or summary yet for Fathom meeting', { externalId }) return null } + + return { + externalId, + title: header.title, + content: formatMeetingContent(header, transcript, summary), + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: header.sourceUrl, + contentHash: header.contentHash, + metadata: { ...header.metadata }, + } }, validateConfig: async ( diff --git a/apps/sim/connectors/fireflies/fireflies.test.ts b/apps/sim/connectors/fireflies/fireflies.test.ts new file mode 100644 index 00000000000..4693726a2b3 --- /dev/null +++ b/apps/sim/connectors/fireflies/fireflies.test.ts @@ -0,0 +1,247 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ FirefliesIcon: () => null })) + +import { firefliesConnector } from '@/connectors/fireflies/fireflies' + +interface GraphQLCall { + query: string + variables: Record +} + +/** Replays the given GraphQL bodies in order and records what was sent. */ +function mockGraphQL(responses: { status?: number; body: unknown }[]) { + const calls: GraphQLCall[] = [] + let index = 0 + mockFetchWithRetry.mockImplementation(async (_url: string, options: RequestInit) => { + calls.push(JSON.parse(String(options.body))) + const route = responses[Math.min(index++, responses.length - 1)] + const status = route.status ?? 200 + return { + ok: status >= 200 && status < 300, + status, + json: async () => route.body, + text: async () => JSON.stringify(route.body), + } as unknown as Response + }) + return calls +} + +function transcript(id: string, extra: Record = {}) { + return { + id, + title: `Meeting ${id}`, + date: 1720476826660, + duration: 45, + organizer_email: 'organizer@example.com', + participants: ['a@example.com'], + transcript_url: `https://app.fireflies.ai/view/${id}`, + speakers: [{ name: 'Ada' }], + ...extra, + } +} + +function page(count: number, offset = 0) { + return { + body: { + data: { + transcripts: Array.from({ length: count }, (_, i) => transcript(`t${offset + i}`)), + }, + }, + } +} + +describe('fireflies listDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes limit/skip/toDate as GraphQL variables and pins the ceiling across pages', async () => { + const calls = mockGraphQL([page(50), page(1, 50)]) + const syncContext: Record = {} + + const first = await firefliesConnector.listDocuments('key', {}, undefined, syncContext) + expect(first.hasMore).toBe(true) + expect(first.nextCursor).toBe('50') + + await firefliesConnector.listDocuments('key', {}, first.nextCursor, syncContext) + + expect(calls[0].variables.limit).toBe(50) + expect(calls[0].variables.skip).toBe(0) + expect(calls[1].variables.skip).toBe(50) + expect(calls[0].variables.toDate).toBe(calls[1].variables.toDate) + expect(typeof calls[0].variables.toDate).toBe('string') + expect(calls[0].query).not.toContain('50') + }) + + it('leaves listingCapped unset when the source is genuinely exhausted', async () => { + mockGraphQL([page(3)]) + const syncContext: Record = {} + + const result = await firefliesConnector.listDocuments('key', {}, undefined, syncContext) + + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('leaves listingCapped unset when maxTranscripts lands exactly on exhaustion', async () => { + mockGraphQL([page(3)]) + const syncContext: Record = {} + + const result = await firefliesConnector.listDocuments( + 'key', + { maxTranscripts: '3' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(3) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxTranscripts hides still-existing transcripts', async () => { + mockGraphQL([page(4)]) + const syncContext: Record = {} + + const result = await firefliesConnector.listDocuments( + 'key', + { maxTranscripts: '3' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(3) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('throws on a GraphQL errors[] payload rather than reporting an empty listing', async () => { + mockGraphQL([ + { body: { data: {}, errors: [{ message: 'Rate limited', code: 'too_many_requests' }] } }, + ]) + + await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow( + /too_many_requests/ + ) + }) + + it('throws rather than reporting an empty listing when a 200 body is unreadable', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: true, + status: 200, + json: async () => { + throw new SyntaxError('Unexpected token < in JSON at position 0') + }, + text: async () => 'gateway', + } as unknown as Response) + const syncContext: Record = {} + + await expect( + firefliesConnector.listDocuments('key', {}, undefined, syncContext) + ).rejects.toThrow(/malformed/i) + }) + + it('throws rather than reporting an empty listing when a 200 carries no data', async () => { + mockGraphQL([{ body: {} }]) + + await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow( + /malformed/i + ) + }) + + it('surfaces the errors[] message on a non-2xx response', async () => { + mockGraphQL([ + { status: 403, body: { errors: [{ message: 'Upgrade required', code: 'paid_required' }] } }, + ]) + + await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow( + /Upgrade required/ + ) + }) +}) + +describe('fireflies getDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('declares the transcript id as String! and reuses the stub contentHash', async () => { + const calls = mockGraphQL([ + page(1), + { + body: { + data: { + transcript: transcript('t0', { + sentences: [{ speaker_name: 'Ada', text: 'Hello' }], + summary: { overview: 'An overview', keywords: ['alpha'] }, + }), + }, + }, + }, + ]) + + const listed = await firefliesConnector.listDocuments('key', {}, undefined, {}) + const stub = listed.documents[0] + const full = await firefliesConnector.getDocument('key', {}, 't0') + + expect(calls[1].query).toContain('$id: String!') + expect(calls[1].variables).toEqual({ id: 't0' }) + expect(full?.contentHash).toBe(stub.contentHash) + expect(stub.contentDeferred).toBe(true) + expect(full?.contentDeferred).toBe(false) + expect(full?.content).toContain('Ada: Hello') + expect(full?.content).toContain('An overview') + }) + + it('renders duration as minutes, not seconds', async () => { + mockGraphQL([{ body: { data: { transcript: transcript('t0', { duration: 45 }) } } }]) + + const full = await firefliesConnector.getDocument('key', {}, 't0') + + expect(full?.content).toContain('Duration: 45 minutes') + expect(full?.metadata?.duration).toBe(45) + }) + + it('falls back to organizer_email when the deprecated host_email is absent', async () => { + mockGraphQL([{ body: { data: { transcript: transcript('t0') } } }]) + + const full = await firefliesConnector.getDocument('key', {}, 't0') + + expect(full?.metadata?.hostEmail).toBe('organizer@example.com') + expect(firefliesConnector.mapTags?.(full?.metadata ?? {}).hostEmail).toBe( + 'organizer@example.com' + ) + }) + + it('returns null when the transcript is not found', async () => { + mockGraphQL([ + { status: 404, body: { errors: [{ message: 'Not found', code: 'object_not_found' }] } }, + ]) + + await expect(firefliesConnector.getDocument('key', {}, 'missing')).resolves.toBeNull() + }) +}) + +describe('fireflies tags', () => { + it('produces every declared tagDefinition id', () => { + const tags = firefliesConnector.mapTags?.({ + hostEmail: 'host@example.com', + speakers: ['Ada', 'Grace'], + duration: 45, + meetingDate: '2024-07-08T22:13:46.660Z', + }) + + expect(Object.keys(tags ?? {}).sort()).toEqual( + firefliesConnector.tagDefinitions?.map((t) => t.id).sort() + ) + }) +}) diff --git a/apps/sim/connectors/fireflies/fireflies.ts b/apps/sim/connectors/fireflies/fireflies.ts index 38d055fff0d..5f56f0b368d 100644 --- a/apps/sim/connectors/fireflies/fireflies.ts +++ b/apps/sim/connectors/fireflies/fireflies.ts @@ -13,14 +13,16 @@ const TRANSCRIPTS_PER_PAGE = 50 interface FirefliesTranscript { id: string title: string + /** Milliseconds since EPOCH (UTC), per the Fireflies Transcript schema. */ date: number + /** Duration of the audio in **minutes**, per the Fireflies Transcript schema. */ duration: number host_email?: string organizer_email?: string participants?: string[] transcript_url?: string - speakers?: { id: number; name: string }[] - sentences?: { index: number; speaker_name: string; text: string }[] + speakers?: { name: string }[] + sentences?: { speaker_name: string; text: string }[] summary?: { keywords?: string[] action_items?: string @@ -29,6 +31,20 @@ interface FirefliesTranscript { } } +/** + * Carries the Fireflies GraphQL error `code` so callers can tell a genuinely missing + * object (`object_not_found`) from a transient fault (`too_many_requests`, 5xx). + */ +class FirefliesApiError extends Error { + constructor( + message: string, + readonly code?: string + ) { + super(message) + this.name = 'FirefliesApiError' + } +} + /** * Executes a GraphQL query against the Fireflies API. */ @@ -51,18 +67,42 @@ async function firefliesGraphQL( retryOptions ) + /** + * Fireflies reports failures as an `errors` array in the body, and does so on + * non-2xx responses too (`object_not_found` → 404, `too_many_requests` → 429, + * `paid_required` → 403). Read the body first so the caller sees the actual + * reason instead of a bare status code. + */ + const data = (await response.json().catch(() => null)) as { + data?: Record | null + errors?: { message?: string; code?: string }[] + } | null + + const firstError = data?.errors?.[0] + if (firstError) { + const code = firstError.code ? ` (${firstError.code})` : '' + throw new FirefliesApiError( + `Fireflies API error${code}: ${firstError.message || 'Unknown GraphQL error'}`, + firstError.code + ) + } + if (!response.ok) { throw new Error(`Fireflies API HTTP error: ${response.status}`) } - const data = await response.json() - - if (data.errors) { - const message = (data.errors as { message: string }[])[0]?.message || 'Unknown GraphQL error' - throw new Error(`Fireflies API error: ${message}`) + /** + * A 2xx carrying neither `errors` nor a `data` object is unreadable — an + * unparseable body, a truncated response, a proxy interstitial. It must raise + * rather than degrade to an empty result: `listDocuments` would otherwise + * report a confident empty listing and the sync engine would reconcile every + * stored document as deleted. + */ + if (!data || typeof data.data !== 'object' || data.data === null) { + throw new Error('Fireflies API returned a malformed response with no data') } - return data.data as Record + return data.data } /** @@ -80,22 +120,23 @@ function formatTranscriptContent(transcript: FirefliesTranscript): string { } if (transcript.duration) { - const minutes = Math.round(transcript.duration / 60) - parts.push(`Duration: ${minutes} minutes`) + parts.push(`Duration: ${Math.round(transcript.duration)} minutes`) } - if (transcript.host_email) { - parts.push(`Host: ${transcript.host_email}`) + const host = transcript.host_email || transcript.organizer_email + if (host) { + parts.push(`Host: ${host}`) } if (transcript.participants && transcript.participants.length > 0) { parts.push(`Participants: ${transcript.participants.join(', ')}`) } - if (transcript.summary?.overview) { + const overview = transcript.summary?.overview || transcript.summary?.short_summary + if (overview) { parts.push('') parts.push('--- Overview ---') - parts.push(transcript.summary.overview) + parts.push(overview) } if (transcript.summary?.action_items) { @@ -120,6 +161,33 @@ function formatTranscriptContent(transcript: FirefliesTranscript): string { return parts.join('\n') } +/** + * Builds the lightweight document stub shared by `listDocuments` and + * `getDocument`, so the metadata-derived `contentHash` is byte-identical on both + * paths and a hydrated transcript is never seen as changed. + */ +function transcriptToStub(transcript: FirefliesTranscript): ExternalDocument { + const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined + const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? [] + + return { + externalId: transcript.id, + title: transcript.title || 'Untitled Meeting', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: transcript.transcript_url || undefined, + contentHash: `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`, + metadata: { + hostEmail: transcript.host_email || transcript.organizer_email, + duration: transcript.duration, + meetingDate, + participants: transcript.participants, + speakers: speakerNames, + }, + } +} + export const firefliesConnector: ConnectorConfig = { ...firefliesConnectorMeta, @@ -133,17 +201,47 @@ export const firefliesConnector: ConnectorConfig = { const maxTranscripts = sourceConfig.maxTranscripts ? Number(sourceConfig.maxTranscripts) : 0 const skip = cursor ? Number(cursor) : 0 + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + + /** + * `skip` is a raw offset and the API documents no ordering guarantee, so a + * transcript created between two pages shifts the window and silently pushes + * a still-existing transcript past the offset — reconciliation would read + * that absence as a deletion. Pinning `toDate` to the moment the sync started + * freezes the result set for the whole walk. It is an intentional scope + * filter at sync start (never a cap), so it must not set `listingCapped`. + */ + let listingCeiling = syncContext?.firefliesListingCeiling as string | undefined + if (!listingCeiling) { + listingCeiling = new Date().toISOString() + if (syncContext) syncContext.firefliesListingCeiling = listingCeiling + } + + /** + * Under a cap, ask for one row beyond what is still needed. The probe row is + * sliced off before it reaches the sync engine and exists only to tell + * "the cap truncated a larger source" apart from "the cap happened to land + * on the last transcript" — the two demand opposite `listingCapped` answers. + */ + const remaining = maxTranscripts > 0 ? Math.max(0, maxTranscripts - prevFetched) : 0 + const pageSize = + maxTranscripts > 0 ? Math.min(TRANSCRIPTS_PER_PAGE, remaining + 1) : TRANSCRIPTS_PER_PAGE const variables: Record = { - limit: TRANSCRIPTS_PER_PAGE, + limit: pageSize, skip, + toDate: listingCeiling, } if (hostEmail.trim()) { variables.host_email = hostEmail.trim() } - logger.info('Listing Fireflies transcripts', { skip, limit: TRANSCRIPTS_PER_PAGE, hostEmail }) + logger.info('Listing Fireflies transcripts', { + skip, + limit: pageSize, + hostEmailFilter: Boolean(hostEmail.trim()), + }) const data = await firefliesGraphQL( accessToken, @@ -151,11 +249,13 @@ export const firefliesConnector: ConnectorConfig = { $limit: Int $skip: Int $host_email: String + $toDate: DateTime ) { transcripts( limit: $limit skip: $skip host_email: $host_email + toDate: $toDate ) { id title @@ -166,7 +266,6 @@ export const firefliesConnector: ConnectorConfig = { participants transcript_url speakers { - id name } } @@ -174,38 +273,37 @@ export const firefliesConnector: ConnectorConfig = { variables ) - const transcripts = (data.transcripts || []) as FirefliesTranscript[] - - const documents: ExternalDocument[] = transcripts.map((transcript) => { - const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined - const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? [] + const transcripts = ( + Array.isArray(data.transcripts) ? data.transcripts : [] + ) as FirefliesTranscript[] - return { - externalId: transcript.id, - title: transcript.title || 'Untitled Meeting', - content: '', - contentDeferred: true, - mimeType: 'text/plain' as const, - sourceUrl: transcript.transcript_url || undefined, - contentHash: `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`, - metadata: { - hostEmail: transcript.host_email, - duration: transcript.duration, - meetingDate, - participants: transcript.participants, - speakers: speakerNames, - }, - } - }) + const allStubs = transcripts.filter((t) => Boolean(t?.id)).map(transcriptToStub) + const documents = maxTranscripts > 0 ? allStubs.slice(0, remaining) : allStubs - const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length + const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched + + /** + * `listingCapped` blocks the sync engine's deletion reconciliation, so it is + * set only when the cap actually hid transcripts that still exist — either + * the probe row came back, or the page came back full. A cap that lands + * exactly on source exhaustion leaves a short page and stays reconcilable, + * otherwise deleted meetings could never be removed from the KB. + */ + const moreAvailable = allStubs.length > documents.length || transcripts.length === pageSize const hitLimit = maxTranscripts > 0 && totalFetched >= maxTranscripts + if (hitLimit && moreAvailable && syncContext) syncContext.listingCapped = true - const hasMore = !hitLimit && transcripts.length === TRANSCRIPTS_PER_PAGE + const hasMore = !hitLimit && moreAvailable return { documents, + /** + * `skip` is an offset over the raw API result set, so it must advance by the + * rows Fireflies returned — not by the stubs kept. Advancing by the kept + * count would re-request any row dropped for a missing `id`. `hasMore` is + * only ever true on the uncapped path, where nothing is sliced off. + */ nextCursor: hasMore ? String(skip + transcripts.length) : undefined, hasMore, } @@ -230,11 +328,9 @@ export const firefliesConnector: ConnectorConfig = { participants transcript_url speakers { - id name } sentences { - index speaker_name text } @@ -250,37 +346,35 @@ export const firefliesConnector: ConnectorConfig = { ) const transcript = data.transcript as FirefliesTranscript | null - if (!transcript) return null - - const content = formatTranscriptContent(transcript) - const contentHash = `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}` + if (!transcript?.id) return null - const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined - const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? [] + const stub = transcriptToStub(transcript) return { - externalId: transcript.id, - title: transcript.title || 'Untitled Meeting', - content, + ...stub, + content: formatTranscriptContent(transcript), contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: transcript.transcript_url || undefined, - contentHash, metadata: { - hostEmail: transcript.host_email, - duration: transcript.duration, - meetingDate, - participants: transcript.participants, - speakers: speakerNames, + ...stub.metadata, keywords: transcript.summary?.keywords, }, } } catch (error) { + /** + * Only `object_not_found` means the transcript is genuinely gone. Every other + * failure — `too_many_requests`, `paid_required`, transport faults — is rethrown so + * the sync engine records a failed row instead of silently dropping a transcript + * that still exists. + */ + if (error instanceof FirefliesApiError && error.code === 'object_not_found') { + logger.info('Fireflies transcript not found', { externalId }) + return null + } logger.warn('Failed to get Fireflies transcript', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/fireflies/meta.ts b/apps/sim/connectors/fireflies/meta.ts index c0603ad0692..26c1248b033 100644 --- a/apps/sim/connectors/fireflies/meta.ts +++ b/apps/sim/connectors/fireflies/meta.ts @@ -35,7 +35,7 @@ export const firefliesConnectorMeta: ConnectorMeta = { tagDefinitions: [ { id: 'hostEmail', displayName: 'Host Email', fieldType: 'text' }, { id: 'speakers', displayName: 'Speakers', fieldType: 'text' }, - { id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' }, + { id: 'duration', displayName: 'Duration (minutes)', fieldType: 'number' }, { id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' }, ], } diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index 2509ecb0df4..b1d1356354f 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -15,10 +15,23 @@ import { const logger = createLogger('GitHubConnector') const GITHUB_API_URL = 'https://api.github.com' -const BATCH_SIZE = 30 +/** + * The whole filtered tree is already resident in `syncContext`, so a listing page + * costs zero API calls — the page size only bounds how many stubs the sync engine + * accumulates per iteration. The engine stops after `MAX_PAGES` (500) and marks the + * listing truncated, so the page size is what sets the connector's file ceiling: + * 500 x 200 = 100,000, matching the Git Trees API's own 100,000-entry limit. + */ +const BATCH_SIZE = 200 const GIT_SHA_PREFIX = 'git-sha:' const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES const BINARY_SNIFF_BYTES = 8000 +/** + * Recorded on binary blobs so they surface once as a skipped row instead of being + * dropped silently — a dropped file stays an `add` forever and its blob is + * re-downloaded in full on every sync. + */ +const BINARY_SKIP_REASON = 'Binary file was not indexed' /** * Heuristic binary detection: Git treats files containing a NUL byte in the @@ -59,13 +72,21 @@ function parseExtensions(extensions: string): Set | null { } /** - * Checks whether a file path matches the extension filter. + * Checks whether a file path matches the extension filter. The extension is read + * from the basename only — a dot in a directory segment (`docs/v1.2/CHANGELOG`) + * must not be mistaken for the file's extension. + * + * A leading dot still counts, so a dotfile matches its own name as the extension + * (`.gitignore` matches the configured extension `.gitignore`). That is the + * long-standing behavior and the only way to select dotfiles at all; narrowing it + * would drop already-indexed files out of the listing and hard-delete them. */ function matchesExtension(filePath: string, extSet: Set | null): boolean { if (!extSet) return true - const lastDot = filePath.lastIndexOf('.') + const fileName = filePath.slice(filePath.lastIndexOf('/') + 1) + const lastDot = fileName.lastIndexOf('.') if (lastDot === -1) return false - return extSet.has(filePath.slice(lastDot).toLowerCase()) + return extSet.has(fileName.slice(lastDot).toLowerCase()) } interface TreeItem { @@ -78,13 +99,18 @@ interface TreeItem { /** * Fetches the full recursive tree for a branch. + * + * Per https://docs.github.com/en/rest/git/trees the recursive form caps at 100,000 + * entries / 7 MB and sets `truncated: true` when the tree exceeds either limit. A + * truncated tree is a partial listing, so the caller must propagate it as + * `listingCapped`. */ async function fetchTree( accessToken: string, owner: string, repo: string, branch: string -): Promise { +): Promise<{ items: TreeItem[]; truncated: boolean }> { const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1` const response = await fetchWithRetry(url, { @@ -104,11 +130,15 @@ async function fetchTree( const data = await response.json() - if (data.truncated) { + const truncated = Boolean(data.truncated) + if (truncated) { logger.warn('GitHub tree was truncated — some files may be missing', { owner, repo, branch }) } - return (data.tree || []).filter((item: TreeItem) => item.type === 'blob') + return { + items: (data.tree || []).filter((item: TreeItem) => item.type === 'blob'), + truncated, + } } /** @@ -146,9 +176,13 @@ async function fetchBlobContent( return buf.toString('utf8') } /** - * Per https://docs.github.com/en/rest/git/blobs the Blobs API only ever - * returns base64. Refuse to silently persist empty content for an - * unexpected encoding so a sync surfaces the error instead. + * `GET /repos/{owner}/{repo}/git/blobs/{sha}` documents a single response + * encoding: "The `content` in the response will always be Base64 encoded." + * The "Currently, `utf-8` and `base64` are supported" sentence belongs to the + * `encoding` REQUEST parameter of `POST .../git/blobs` (Create a blob) and does + * not describe this response, so no `utf-8` branch is warranted here. Any other + * encoding would silently persist empty content, so it throws and surfaces as a + * failed document instead. */ throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`) } @@ -163,7 +197,7 @@ function treeItemToStub( owner: string, repo: string, branch: string, - item: TreeItem + item: { path: string; sha: string; size?: number } ): ExternalDocument { return { externalId: item.path, @@ -202,7 +236,7 @@ export const githubConnector: ConnectorConfig = { if (syncContext?.filteredTree) { capped = syncContext.filteredTree as TreeItem[] } else { - const tree = await fetchTree(accessToken, owner, repo, branch) + const { items: tree, truncated } = await fetchTree(accessToken, owner, repo, branch) // Filter by path prefix and extensions. Oversized files are kept here and // surfaced as skipped (failed) documents at stub time so they stay visible. @@ -223,6 +257,26 @@ export const githubConnector: ConnectorConfig = { 0 ).documents : filtered + + /** + * The listing is partial whenever the Git Trees API truncated the response or + * `maxFiles` dropped files that still exist in the repo. The sync engine + * hard-deletes every stored document absent from a complete listing, so flag + * `listingCapped` to suppress reconciliation. Path/extension filters are + * intentional scope narrowing and deliberately do NOT set the flag — files + * that leave that scope should reconcile away. + */ + if (syncContext && (truncated || capped.length < filtered.length)) { + syncContext.listingCapped = true + logger.warn('GitHub listing is partial; skipping deletion reconciliation', { + owner, + repo, + branch, + truncated, + matched: filtered.length, + listed: capped.length, + }) + } if (syncContext) syncContext.filteredTree = capped } @@ -279,6 +333,18 @@ export const githubConnector: ConnectorConfig = { if (!response.ok) { if (response.status === 404) return null + /** + * A rate-limit 403 never reaches here: `fetchWithRetry` treats a 403 carrying + * `retry-after` or `x-ratelimit-remaining: 0` as retryable and throws once the + * retries are spent, so it lands in the catch below as a failure. + * + * A 403 that survives is usually an authorization denial, but NOT always: this + * request sends `application/vnd.github+json`, and the Contents API documents + * that files between 1-100 MB support "only the `raw` or `object` custom media + * types". A >1 MB text file therefore also lands here and is dropped, which on + * an `add` is silent (a fulfilled `null` records no failure). Reconciliation is + * unaffected — the file is already in `seenExternalIds` from the listing. + */ if (response.status === 403) { logger.info('Skipping GitHub file rejected by Contents API', { path, @@ -293,30 +359,21 @@ export const githubConnector: ConnectorConfig = { const data = await response.json() const size = typeof data.size === 'number' ? data.size : 0 + // Shared stub keeps externalId, sourceUrl, contentHash, and metadata byte-identical + // to what `listDocuments` produced, so hydration never looks like a content change. + const stub = treeItemToStub(owner, repo, branch, { + path, + sha: data.sha as string, + size, + }) + if (size > MAX_FILE_SIZE) { logger.info('Skipping GitHub file exceeding size limit', { path, size, limit: MAX_FILE_SIZE, }) - return markSkipped( - { - externalId, - title: path.split('/').pop() || path, - content: '', - mimeType: 'text/plain', - sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`, - contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`, - metadata: { - path, - sha: data.sha as string, - size, - branch, - repository: `${owner}/${repo}`, - }, - }, - sizeLimitSkipReason(MAX_FILE_SIZE) - ) + return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) } const rawContent = (data.content as string) || '' @@ -326,14 +383,27 @@ export const githubConnector: ConnectorConfig = { const buf = Buffer.from(rawContent, 'base64') if (isBinaryBuffer(buf)) { logger.info('Skipping binary GitHub file', { path, size }) - return null + return markSkipped(stub, BINARY_SKIP_REASON) } content = buf.toString('utf8') } else if (encoding === 'none' && data.sha && size > 0) { + /** + * Per https://docs.github.com/en/rest/repos/contents, for files of 1-100 MB + * "only the `raw` or `object` custom media types are supported", and it is + * specifically "when using the `object` media type" that "the `content` field + * will be an empty string and the `encoding` field will be `none`". + * + * This request sends `application/vnd.github+json`, so that precondition does + * not hold and this branch is currently unreachable — such files 403 above + * instead. Reaching it would require requesting + * `application/vnd.github.object+json`. The fallback itself is correct: the Git + * Blobs API returns the same blob as JSON and is documented to support blobs up + * to 100 MB. + */ const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string) if (blobContent === null) { logger.info('Skipping binary GitHub file', { path, size }) - return null + return markSkipped(stub, BINARY_SKIP_REASON) } content = blobContent } else { @@ -341,27 +411,22 @@ export const githubConnector: ConnectorConfig = { } return { - externalId, - title: path.split('/').pop() || path, + ...stub, content, contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`, - contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`, - metadata: { - path, - sha: data.sha as string, - size: data.size as number, - branch, - repository: `${owner}/${repo}`, - lastModified: lastModifiedHeader, - }, + metadata: { ...stub.metadata, lastModified: lastModifiedHeader }, } } catch (error) { + /** + * Rethrow so hydration rejects and the sync engine counts a visible `docsFailed` + * row. Returning `null` instead reports a transient GitHub failure as success — + * an already-indexed file is silently counted as unchanged, and a new file + * disappears from the run entirely with nothing recorded. + */ logger.warn(`Failed to fetch GitHub document ${externalId}`, { error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index 99586321f48..e94c3a8d28f 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -113,6 +113,46 @@ function parseNextLink(linkHeader: string | null): string | undefined { return undefined } +/** + * Issues a listing GET, transparently downgrading to offset pagination when the + * GitLab instance rejects keyset pagination. + * + * Keyset support is per-resource and version-gated — GitLab's documented + * pagination table records the repository tree gaining it in 17.1 and project + * issues only in 18.3 — so self-managed hosts on an older release would fail the + * whole sync. The request is therefore retried once without `pagination=keyset`; + * offset pagination emits the same `Link: rel="next"` header the caller already + * follows, so nothing downstream changes. + * + * The `405` trigger is NOT documented. It comes from GitLab's own + * `lib/api/helpers/pagination_strategies.rb`, which raises + * `error!('Keyset pagination is not yet available for this type of request', 405)` + * when the relation/order combination has no keyset strategy. Treat it as + * observed behavior that could change without a docs-visible deprecation; the + * fallback is written to be a no-op when the URL carries no `pagination` param, + * so an unrelated 405 is passed straight through. + */ +async function fetchListing(url: string, accessToken: string): Promise { + const response = await secureFetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) + if (response.status !== 405) return response + + let offsetUrl: string + try { + const parsed = new URL(url) + if (!parsed.searchParams.has('pagination')) return response + parsed.searchParams.delete('pagination') + offsetUrl = parsed.toString() + } catch { + return response + } + + logger.warn('GitLab rejected keyset pagination; retrying with offset pagination', { url }) + return secureFetchWithRetry(offsetUrl, { method: 'GET', headers: authHeaders(accessToken) }) +} + /** * Returns the ordered list of active sync phases for a content-type choice. */ @@ -197,9 +237,22 @@ function buildApiBase(host: string): string { /** * Returns the encoded project identifier (numeric ID or URL-encoded path). * GitLab accepts a numeric ID or the URL-encoded `group/project` path. + * + * Decoding first makes `group/project` and an already-encoded + * `group%2Fproject` converge on the same single-encoded result — without it a + * pasted `%2F` becomes `%252F`, which GitLab decodes once to the literal string + * `%2F` and the project lookup 404s. Mirrors `encodeGitLabResourceId` in the + * GitLab tools. */ function encodeProjectId(project: unknown): string { - return encodeURIComponent(String(project ?? '').trim()) + const raw = String(project ?? '').trim() + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Not a valid percent-encoding (e.g. a bare `%`) — treat as already raw. + } + return encodeURIComponent(decoded) } /** @@ -332,9 +385,15 @@ function fileToDocument( if (!blobSha) return null const title = path.split('/').pop() || path - const skippedForSize = (size: number): ExternalDocument => { - logger.info('Skipping oversized GitLab file', { path, size }) - return markSkipped( + /** + * Returns the file as an explicitly skipped document rather than dropping it. + * A dropped (`null`) file is never stored, so the next sync lists it again and + * re-downloads the same blob forever; a skipped document carries the blob-SHA + * hash, so it surfaces once as a failed row and is not re-fetched until the + * file actually changes. + */ + const skipped = (reason: string, size: number): ExternalDocument => + markSkipped( { externalId: `${FILE_PREFIX}${path}`, title, @@ -344,22 +403,23 @@ function fileToDocument( contentHash: buildFileContentHash(encodedProject, path, blobSha), metadata: { contentType: 'file', title, path, size }, }, - sizeLimitSkipReason(MAX_FILE_SIZE) + reason ) - } if (typeof file.size === 'number' && file.size > MAX_FILE_SIZE) { - return skippedForSize(file.size) + logger.info('Skipping oversized GitLab file', { path, size: file.size }) + return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), file.size) } const raw = typeof file.content === 'string' ? file.content : '' const buffer = file.encoding === 'base64' ? Buffer.from(raw, 'base64') : Buffer.from(raw, 'utf8') if (isBinaryBuffer(buffer)) { logger.info('Skipping binary GitLab file', { path }) - return null + return skipped('Binary file was not indexed', buffer.byteLength) } if (buffer.byteLength > MAX_FILE_SIZE) { - return skippedForSize(buffer.byteLength) + logger.info('Skipping oversized GitLab file', { path, size: buffer.byteLength }) + return skipped(sizeLimitSkipReason(MAX_FILE_SIZE), buffer.byteLength) } const content = buffer.toString('utf8') @@ -492,6 +552,49 @@ async function fetchProject( ) } +/** + * Resolves the project's `group/project` path, used to build web UI source URLs. + * Cached on syncContext so listing pages and deferred `getDocument` hydration in + * the same run share one lookup. + * + * Throws when the project itself cannot be read. That is not a cosmetic failure: + * GitLab collapses "not authorized to read" into `404 Not Found` on read + * endpoints, so a token that has lost access would otherwise produce an empty + * but apparently successful listing and let deletion reconciliation hard-delete + * every previously synced document. Failing here also establishes that the + * project is visible, which is what lets the per-phase 404 handling below be read + * as "this ref/wiki is genuinely absent" rather than "access was revoked". + * + * Returns '' only when the project record itself carries no + * `path_with_namespace`, in which case callers fall back to API source URLs. + */ +async function resolveProjectPath( + syncContext: Record | undefined, + apiBase: string, + encodedProject: string, + accessToken: string +): Promise { + const cached = syncContext?.projectPath + if (typeof cached === 'string' && cached) return cached + + const response = await fetchProject(apiBase, encodedProject, accessToken) + if (!response.ok) { + throw new Error( + `Cannot access GitLab project ${encodedProject}: ${response.status}. On GitLab a 404 also means the token is no longer authorized to read it.` + ) + } + + const project = (await response.json()) as GitLabProject + const path = project.path_with_namespace ?? '' + if (syncContext) { + if (path) syncContext.projectPath = path + if (project.default_branch && !syncContext.defaultBranch) { + syncContext.defaultBranch = project.default_branch + } + } + return path +} + /** * Encodes the listing cursor. The cursor packs the resource phase (repo ➜ wiki ➜ * issues) and a per-phase continuation token so a single sync walks the phases in @@ -613,18 +716,7 @@ export const gitlabConnector: ConnectorConfig = { const phases = activePhases(choice) if (phases.length === 0) return { documents: [], hasMore: false } - let projectPath = (syncContext?.projectPath as string) ?? '' - if (!projectPath && syncContext) { - const projectResponse = await fetchProject(apiBase, encodedProject, accessToken) - if (projectResponse.ok) { - const project = (await projectResponse.json()) as GitLabProject - projectPath = project.path_with_namespace ?? '' - syncContext.projectPath = projectPath - if (project.default_branch && !syncContext.defaultBranch) { - syncContext.defaultBranch = project.default_branch - } - } - } + const projectPath = await resolveProjectPath(syncContext, apiBase, encodedProject, accessToken) let state = decodeCursor(cursor, phases[0]) if (!phases.includes(state.phase)) state = { phase: phases[0], issuePage: 1 } @@ -662,13 +754,18 @@ export const gitlabConnector: ConnectorConfig = { continued: Boolean(state.fileNextUrl), }) - const response = await secureFetchWithRetry(url, { - method: 'GET', - headers: authHeaders(accessToken), - }) + const response = await fetchListing(url, accessToken) if (!response.ok) { - if (response.status === 404 || response.status === 403) { + if (response.status === 401 || response.status === 403 || response.status === 404) { + /** + * 401/403 mean the token stopped working mid-sync — flag the listing as + * incomplete so deletion reconciliation does not hard-delete previously + * synced files. A 404 here is safe to reconcile against precisely because + * `resolveProjectPath` already proved the project is readable, so the only + * remaining meaning is that the ref or repository is absent. + */ + if (response.status !== 404 && syncContext) syncContext.listingCapped = true logger.warn('GitLab repository tree unavailable; skipping files', { host, project: encodedProject, @@ -724,7 +821,16 @@ export const gitlabConnector: ConnectorConfig = { }) if (!response.ok) { - if (response.status === 403 || response.status === 404) { + if (response.status === 401 || response.status === 403 || response.status === 404) { + /** + * 401/403 mean the token stopped working mid-sync — flag the listing as + * incomplete so deletion reconciliation does not hard-delete previously + * synced wiki pages. A 404 here is safe to reconcile against precisely + * because `resolveProjectPath` already proved the project is readable, so + * the only remaining meaning is that the wiki feature or its content is + * absent. + */ + if (response.status !== 404 && syncContext) syncContext.listingCapped = true logger.warn('GitLab wiki unavailable; skipping wiki phase', { host, project: encodedProject, @@ -793,10 +899,7 @@ export const gitlabConnector: ConnectorConfig = { incremental: Boolean(lastSyncAt), }) - const response = await secureFetchWithRetry(url, { - method: 'GET', - headers: authHeaders(accessToken), - }) + const response = await fetchListing(url, accessToken) if (!response.ok) { const errorText = await response.text().catch(() => '') @@ -848,9 +951,14 @@ export const gitlabConnector: ConnectorConfig = { const encodedProject = encodeProjectId(sourceConfig.project) if (!encodedProject || !externalId) return null - const projectPath = (syncContext?.projectPath as string) ?? '' - try { + const projectPath = await resolveProjectPath( + syncContext, + apiBase, + encodedProject, + accessToken + ) + if (externalId.startsWith(WIKI_PREFIX)) { const slug = externalId.slice(WIKI_PREFIX.length) if (!slug) return null @@ -920,10 +1028,17 @@ export const gitlabConnector: ConnectorConfig = { return null } catch (error) { + /** + * Only the 404 checks above (and an unrecognized externalId prefix) mean the object + * is genuinely gone. Every other failure is rethrown so the sync engine records a + * visible `docsFailed` row. Returning `null` instead would report a transient + * GitLab fault as success — an already-indexed document is silently counted as + * unchanged, and a new one vanishes from the run with nothing recorded. + */ logger.warn(`Failed to fetch GitLab document ${externalId}`, { error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/gitlab/meta.ts b/apps/sim/connectors/gitlab/meta.ts index 720121ec79d..9490a50e823 100644 --- a/apps/sim/connectors/gitlab/meta.ts +++ b/apps/sim/connectors/gitlab/meta.ts @@ -6,7 +6,7 @@ export const gitlabConnectorMeta: ConnectorMeta = { name: 'GitLab', description: 'Sync repository files, wiki pages, and issues from a GitLab project into your knowledge base', - version: '1.0.0', + version: '1.1.0', icon: GitLabIcon, /** @@ -52,7 +52,9 @@ export const gitlabConnectorMeta: ConnectorMeta = { { label: 'Issues only', id: 'issues' }, { label: 'Wiki & Issues', id: 'both' }, ], - description: 'Which content to index. "Code" syncs repository files (READMEs, docs, source).', + placeholder: 'Wiki & Issues', + description: + 'Which content to index. "Code" syncs repository files (READMEs, docs, source). Defaults to Wiki & Issues when left unset.', }, { id: 'ref', diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index eab76991b75..4e604f1115f 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -17,6 +17,7 @@ interface GmailHeader { interface GmailMessagePart { mimeType?: string + filename?: string body?: { data?: string; size?: number } parts?: GmailMessagePart[] headers?: GmailHeader[] @@ -44,6 +45,79 @@ interface GmailLabel { type?: string } +const LABEL_CACHE_KEY = '_gmailLabelCache' + +interface GmailLabelIndex { + /** Label id (e.g. `INBOX`, `Label_7`) to display name. */ + byId: Record + /** Lowercased display name to label id. */ + idByLowerName: Record +} + +const EMPTY_LABEL_INDEX: GmailLabelIndex = { byId: {}, idByLowerName: {} } + +function buildLabelIndex(labels: GmailLabel[]): GmailLabelIndex { + const index: GmailLabelIndex = { byId: {}, idByLowerName: {} } + for (const label of labels) { + if (!label?.id || typeof label.name !== 'string') continue + index.byId[label.id] = label.name + index.idByLowerName[label.name.toLowerCase()] = label.id + } + return index +} + +/** + * Fetches `users.labels.list` once and caches the result on `syncContext` so it is + * shared across pages and across every deferred `getDocument` hydration. A failed + * fetch resolves to `null` and that failure is cached too, so a persistently + * failing labels call cannot turn into a per-document API call. + * + * Callers must distinguish `null` from an empty index: label tagging degrades to + * raw ids, but query building cannot — see `listDocuments`. + */ +async function getLabelIndex( + accessToken: string, + syncContext?: Record +): Promise { + if (syncContext && LABEL_CACHE_KEY in syncContext) { + return syncContext[LABEL_CACHE_KEY] as GmailLabelIndex | null + } + + let index: GmailLabelIndex | null = null + try { + const response = await fetchWithRetry(`${GMAIL_API_BASE}/labels`, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + + if (response.ok) { + const data = await response.json() + index = buildLabelIndex((data.labels || []) as GmailLabel[]) + } else { + logger.warn('Failed to fetch Gmail labels', { status: response.status }) + } + } catch (error) { + logger.warn('Failed to fetch Gmail labels', { error: toError(error).message }) + } + + if (syncContext) syncContext[LABEL_CACHE_KEY] = index + return index +} + +/** + * Resolves a configured label value to the display name the `label:` search + * operator matches on. The `gmail.labels` selector stores label **ids** + * (`Label_7`), while the search operator only understands label **names**, so an + * id is translated through the label index. Values that are already names (typed + * into the advanced input) pass through unchanged. + */ +function resolveLabelName(value: string, index: GmailLabelIndex): string { + return index.byId[value] ?? value +} + /** * Formats a single Gmail label name for use in a `label:` operator. * Gmail search syntax accepts quoted strings for labels containing spaces; @@ -64,10 +138,15 @@ function formatLabelToken(name: string): string { * Combines the user's custom query with the label and date range filters. * When multiple labels are provided, they are OR-joined: `(label:A OR label:B)`. */ -function buildSearchQuery(sourceConfig: Record): string { +function buildSearchQuery( + sourceConfig: Record, + labelIndex: GmailLabelIndex = EMPTY_LABEL_INDEX +): string { const parts: string[] = [] - const labelNames = parseMultiValue(sourceConfig.label) + const labelNames = parseMultiValue(sourceConfig.label).map((value) => + resolveLabelName(value, labelIndex) + ) if (labelNames.length === 1) { const token = formatLabelToken(labelNames[0]) if (token) parts.push(token) @@ -147,30 +226,47 @@ function decodeBase64Url(data: string): string { return Buffer.from(data, 'base64url').toString('utf-8') } +/** + * True when a MIME part is an attachment rather than a body part, so a `.txt` or + * `.html` attachment is never mistaken for the message body. + * + * `MessagePart.filename` is documented as "the filename of the attachment. Only + * present if this message part represents an attachment" — i.e. absent on body + * parts. In practice Gmail also emits `""` there, so the truthiness test covers + * both the documented and the observed shape. + */ +function isAttachmentPart(part: GmailMessagePart): boolean { + return Boolean(part.filename) +} + /** * Extracts the plain text body from a Gmail message payload. - * Prefers text/plain, falls back to text/html with tag stripping. + * Prefers text/plain, falls back to text/html with tag stripping, and recurses + * through nested multiparts (e.g. a multipart/alternative inside a multipart/mixed). */ function extractBody(part: GmailMessagePart): string { + if (isAttachmentPart(part)) return '' + if (part.mimeType === 'text/plain' && part.body?.data) { return decodeBase64Url(part.body.data) } if (part.parts) { + const children = part.parts.filter((child) => !isAttachmentPart(child)) // Prefer text/plain from multipart - for (const child of part.parts) { + for (const child of children) { if (child.mimeType === 'text/plain' && child.body?.data) { return decodeBase64Url(child.body.data) } } // Fall back to text/html - for (const child of part.parts) { + for (const child of children) { if (child.mimeType === 'text/html' && child.body?.data) { return htmlToPlainText(decodeBase64Url(child.body.data)) } } // Recurse into nested multipart - for (const child of part.parts) { + for (const child of children) { const result = extractBody(child) if (result) return result } @@ -210,7 +306,16 @@ function formatThread(thread: GmailThread): { const subject = getHeader(firstMessage.payload, 'Subject') || 'No Subject' const from = getHeader(firstMessage.payload, 'From') || 'Unknown' const to = getHeader(firstMessage.payload, 'To') || '' - const labelIds = firstMessage.labelIds || [] + /** + * Gmail applies labels per message, not per thread — the thread's label set is + * the union across its messages. Reading only `messages[0]` drops labels that + * were applied to a later reply (and are exactly what a `label:` filter matched). + */ + const labelIdSet = new Set() + for (const msg of messages) { + for (const id of msg.labelIds ?? []) labelIdSet.add(id) + } + const labelIds = [...labelIdSet] const lines: string[] = [] lines.push(`Subject: ${subject}`) @@ -255,7 +360,7 @@ function formatThread(thread: GmailThread): { * Fetches a full thread with all its messages. */ async function fetchThread(accessToken: string, threadId: string): Promise { - const url = `${GMAIL_API_BASE}/threads/${threadId}?format=FULL` + const url = `${GMAIL_API_BASE}/threads/${encodeURIComponent(threadId)}?format=full` const response = await fetchWithRetry(url, { method: 'GET', @@ -274,43 +379,16 @@ async function fetchThread(accessToken: string, threadId: string): Promise ): Promise { - const cacheKey = '_gmailLabelCache' - - if (syncContext && !syncContext[cacheKey]) { - try { - const url = `${GMAIL_API_BASE}/labels` - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (response.ok) { - const data = await response.json() - const labels = (data.labels || []) as GmailLabel[] - const labelMap: Record = {} - for (const label of labels) { - labelMap[label.id] = label.name - } - syncContext[cacheKey] = labelMap - } - } catch { - syncContext[cacheKey] = {} - } - } - - const cache = (syncContext?.[cacheKey] as Record) ?? {} + const index = await getLabelIndex(accessToken, syncContext) return labelIds - .map((id) => cache[id] || id) + .map((id) => index?.byId[id] || id) .filter((name) => !name.startsWith('CATEGORY_') && name !== 'UNREAD') } @@ -329,12 +407,21 @@ function threadToStub(thread: { content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`, + sourceUrl: threadUrl(thread.id), contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`, metadata: {}, } } +/** + * Deep link to a thread. `#all` is used rather than `#inbox` because a synced + * thread may be archived or live only under a user label, where an `#inbox` + * fragment resolves to nothing. + */ +function threadUrl(threadId: string): string { + return `https://mail.google.com/mail/u/0/#all/${threadId}` +} + export const gmailConnector: ConnectorConfig = { ...gmailConnectorMeta, @@ -344,7 +431,21 @@ export const gmailConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { - const searchQuery = buildSearchQuery(sourceConfig) + let labelIndex = EMPTY_LABEL_INDEX + if (parseMultiValue(sourceConfig.label).length > 0) { + /** + * Configured labels are ids that only the label index can turn into the + * names `label:` matches. Without it the query silently matches nothing and + * the sync would report a complete, empty listing — which the engine reads + * as "every stored thread was deleted". Fail the sync instead. + */ + const resolved = await getLabelIndex(accessToken, syncContext) + if (!resolved) { + throw new Error('Failed to fetch Gmail labels; cannot resolve the configured label filter') + } + labelIndex = resolved + } + const searchQuery = buildSearchQuery(sourceConfig, labelIndex) const maxThreads = sourceConfig.maxThreads ? Number(sourceConfig.maxThreads) : DEFAULT_MAX_THREADS @@ -393,20 +494,30 @@ export const gmailConnector: ConnectorConfig = { const data = await response.json() const threads = (data.threads || []) as { id: string; snippet?: string; historyId?: string }[] - - if (threads.length === 0) { - return { documents: [], hasMore: false } - } + const nextPageToken = data.nextPageToken as string | undefined const documents = threads.map(threadToStub) const newTotal = totalFetched + documents.length if (syncContext) syncContext.totalThreadsFetched = newTotal - const nextPageToken = data.nextPageToken as string | undefined const hitLimit = newTotal >= maxThreads - if (hitLimit && syncContext) syncContext.listingCapped = true + /** + * Only a cap that actually truncates a longer listing blocks deletion + * reconciliation. Reaching the cap exactly as the source runs out + * (`nextPageToken` absent) is genuine exhaustion, and flagging it would + * permanently prevent deleted threads from being reconciled. + */ + if (hitLimit && nextPageToken && syncContext) syncContext.listingCapped = true + + /** + * `nextPageToken` is the only exhaustion signal. `users.threads.list` documents + * the token as the way to reach the next page but never guarantees a non-empty + * `threads` array alongside one, so an empty page is not treated as the end: + * doing so would report a complete-but-empty listing and let the sync engine + * hard-delete every previously stored thread. + */ return { documents, nextCursor: hitLimit ? undefined : nextPageToken, @@ -420,33 +531,24 @@ export const gmailConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { - try { - const thread = await fetchThread(accessToken, externalId) - if (!thread) return null - - const { content, subject, metadata } = formatThread(thread) - if (!content.trim()) return null - - const labelIds = (metadata.labelIds as string[]) || [] - const labelNames = await resolveLabelNames(accessToken, labelIds, syncContext) - metadata.labels = labelNames - - return { - externalId: thread.id, - title: subject, - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`, - contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`, - metadata, - } - } catch (error) { - logger.warn('Failed to get Gmail thread', { - externalId, - error: toError(error).message, - }) - return null + const thread = await fetchThread(accessToken, externalId) + if (!thread) return null + + const { content, subject, metadata } = formatThread(thread) + if (!content.trim()) return null + + const labelIds = (metadata.labelIds as string[]) || [] + metadata.labels = await resolveLabelNames(accessToken, labelIds, syncContext) + + return { + externalId: thread.id, + title: subject, + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: threadUrl(thread.id), + contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`, + metadata, } }, @@ -479,9 +581,14 @@ export const gmailConnector: ConnectorConfig = { return { valid: false, error: `Failed to access Gmail: ${profileResponse.status}` } } - // If labels are specified, verify each one exists - const labelNames = parseMultiValue(sourceConfig.label) - if (labelNames.length > 0) { + /** + * Labels may arrive as ids (from the `gmail.labels` selector) or as names + * (typed into the advanced input), so both forms are accepted here and the + * same index is what `buildSearchQuery` resolves ids through. + */ + const configuredLabels = parseMultiValue(sourceConfig.label) + let labelIndex = EMPTY_LABEL_INDEX + if (configuredLabels.length > 0) { const labelsUrl = `${GMAIL_API_BASE}/labels` const labelsResponse = await fetchWithRetry( labelsUrl, @@ -501,8 +608,10 @@ export const gmailConnector: ConnectorConfig = { const labelsData = await labelsResponse.json() const labels = (labelsData.labels || []) as GmailLabel[] - const labelNameSet = new Set(labels.map((l) => l.name.toLowerCase())) - const missing = labelNames.filter((name) => !labelNameSet.has(name.toLowerCase())) + labelIndex = buildLabelIndex(labels) + const missing = configuredLabels.filter( + (value) => !labelIndex.byId[value] && !labelIndex.idByLowerName[value.toLowerCase()] + ) if (missing.length > 0) { return { @@ -523,7 +632,7 @@ export const gmailConnector: ConnectorConfig = { // If a custom query is specified, verify it's valid by doing a dry-run const query = sourceConfig.query as string | undefined if (query?.trim()) { - const searchQuery = buildSearchQuery(sourceConfig) + const searchQuery = buildSearchQuery(sourceConfig, labelIndex) const testUrl = `${GMAIL_API_BASE}/threads?q=${encodeURIComponent(searchQuery)}&maxResults=1` const testResponse = await fetchWithRetry( testUrl, diff --git a/apps/sim/connectors/gong/gong.ts b/apps/sim/connectors/gong/gong.ts index 80b2157a05d..77ed6655aa3 100644 --- a/apps/sim/connectors/gong/gong.ts +++ b/apps/sim/connectors/gong/gong.ts @@ -1,9 +1,9 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { gongConnectorMeta } from '@/connectors/gong/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { joinTagArray, parseTagDate } from '@/connectors/utils' const logger = createLogger('GongConnector') @@ -211,22 +211,89 @@ function buildMetadata( } } +/** + * Everything about a call that is derivable from the listing response alone. + * Cached in `syncContext` by `listDocuments` so the deferred-content fetch in + * `getDocument` does not have to re-request `/v2/calls/extensive` for metadata + * it already had — halving the request volume against Gong's 3 req/s quota. + */ +interface GongCallHeader { + id: string + title: string + sourceUrl?: string + contentHash: string + started?: string + duration?: number + participants: string[] + speakerMap: Record + metadata: Record +} + +/** + * Upper bound on cached call headers. A sync lists every call before hydrating + * any of them, so an uncapped cache would hold the whole workspace in memory. + * Headers past the cap simply miss and fall back to a per-call API request. + */ +const CALL_HEADER_CACHE_LIMIT = 5000 + +function buildCallHeader(call: GongExtensiveCall): GongCallHeader | null { + const metaData = call.metaData + const callId = metaData?.id + if (!callId) return null + const participants = buildParticipantNames(call.parties) + return { + id: callId, + title: buildCallTitle(metaData), + sourceUrl: metaData?.url || undefined, + contentHash: buildContentHash(callId, metaData?.started), + started: metaData?.started, + duration: metaData?.duration, + participants, + speakerMap: buildSpeakerMap(call.parties), + metadata: buildMetadata(metaData, participants), + } +} + +/** + * Reads the cached header for a call out of the shared sync context. + */ +function readCachedHeader( + syncContext: Record | undefined, + callId: string +): GongCallHeader | undefined { + const cache = syncContext?.gongCallHeaders as Map | undefined + return cache?.get(callId) +} + +/** + * Stores the header for a call in the shared sync context, up to the cache cap. + */ +function cacheHeader( + syncContext: Record | undefined, + header: GongCallHeader +): void { + if (!syncContext) return + let cache = syncContext.gongCallHeaders as Map | undefined + if (!cache) { + cache = new Map() + syncContext.gongCallHeaders = cache + } + if (cache.size >= CALL_HEADER_CACHE_LIMIT && !cache.has(header.id)) return + cache.set(header.id, header) +} + /** * Formats a call's transcript into speaker-attributed plain text with a header * describing the call (title, date, duration, participants). */ -function formatTranscriptContent( - metaData: GongCallMetaData | undefined, - participants: string[], - speakerMap: Record, - monologues: GongMonologue[] -): string { +function formatTranscriptContent(header: GongCallHeader, monologues: GongMonologue[]): string { const parts: string[] = [] + const { participants, speakerMap } = header - parts.push(`Call: ${buildCallTitle(metaData)}`) - if (metaData?.started) parts.push(`Date: ${metaData.started}`) - if (metaData?.duration != null) { - const minutes = Math.round(metaData.duration / 60) + parts.push(`Call: ${header.title}`) + if (header.started) parts.push(`Date: ${header.started}`) + if (header.duration != null) { + const minutes = Math.round(header.duration / 60) parts.push(`Duration: ${minutes} minutes`) } if (participants.length > 0) parts.push(`Participants: ${participants.join(', ')}`) @@ -295,6 +362,14 @@ async function fetchExtensiveCalls( retryOptions ) + /** + * Gong answers a filter that matches no calls with `404 No calls found for the + * specified period` rather than an empty list — the documented 404 for + * `/v2/calls/extensive`. That is an ordinary outcome for a narrow incremental + * window, so it maps to an empty page instead of failing the sync. + */ + if (response.status === 404) return { calls: [] } + if (!response.ok) { const errorText = await response.text().catch(() => '') throw new Error( @@ -348,18 +423,18 @@ export const gongConnector: ConnectorConfig = { const allDocuments: ExternalDocument[] = [] for (const call of calls) { - const callId = call.metaData?.id - if (!callId) continue - const participants = buildParticipantNames(call.parties) + const header = buildCallHeader(call) + if (!header) continue + cacheHeader(syncContext, header) allDocuments.push({ - externalId: callId, - title: buildCallTitle(call.metaData), + externalId: header.id, + title: header.title, content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: call.metaData?.url || undefined, - contentHash: buildContentHash(callId, call.metaData?.started), - metadata: buildMetadata(call.metaData, participants), + sourceUrl: header.sourceUrl, + contentHash: header.contentHash, + metadata: header.metadata, }) } @@ -398,74 +473,77 @@ export const gongConnector: ConnectorConfig = { } }, + /** + * Hydrates a listing stub with its transcript. + * + * Returns `null` only when the call genuinely has nothing to index yet (call + * gone, transcript still processing). Transport, rate-limit, and server errors + * propagate so the sync engine records them as failed documents instead of + * reporting a clean sync that silently dropped calls. + */ getDocument: async ( accessToken: string, sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { - try { - if (!externalId) return null + if (!externalId) return null + let header = readCachedHeader(syncContext, externalId) + if (!header) { const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim() const filter: Record = { callIds: [externalId] } if (workspaceId) filter.workspaceId = workspaceId const callData = await fetchExtensiveCalls(accessToken, filter, undefined) const call = callData.calls?.[0] - if (!call?.metaData?.id) { + const fetchedHeader = call ? buildCallHeader(call) : null + if (!fetchedHeader) { logger.warn('Gong call not found', { externalId }) return null } + header = fetchedHeader + cacheHeader(syncContext, header) + } - const metaData = call.metaData - const participants = buildParticipantNames(call.parties) - const speakerMap = buildSpeakerMap(call.parties) - - const transcriptResponse = await fetchWithRetry(`${GONG_API_BASE}/calls/transcript`, { - method: 'POST', - headers: buildHeaders(accessToken), - body: JSON.stringify({ filter: { callIds: [externalId] } }), - }) - - if (!transcriptResponse.ok) { - if (transcriptResponse.status === 404) return null - throw new Error(`Failed to fetch Gong transcript: ${transcriptResponse.status}`) - } - - const transcriptData = (await transcriptResponse.json()) as GongTranscriptResponse - const callTranscript = transcriptData.callTranscripts?.find( - (entry) => entry.callId === externalId - ) - const monologues = callTranscript?.transcript ?? [] - if (monologues.length === 0) { - logger.info('Transcript not available for Gong call', { externalId }) - return null - } + const transcriptResponse = await fetchWithRetry(`${GONG_API_BASE}/calls/transcript`, { + method: 'POST', + headers: buildHeaders(accessToken), + body: JSON.stringify({ filter: { callIds: [externalId] } }), + }) - const hasSpokenText = monologues.some((monologue) => - (monologue.sentences ?? []).some((sentence) => Boolean(sentence.text?.trim())) + if (!transcriptResponse.ok) { + if (transcriptResponse.status === 404) return null + const errorText = await transcriptResponse.text().catch(() => '') + throw new Error( + `Failed to fetch Gong transcript: ${transcriptResponse.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}` ) - if (!hasSpokenText) return null + } - const content = formatTranscriptContent(metaData, participants, speakerMap, monologues) + const transcriptData = (await transcriptResponse.json()) as GongTranscriptResponse + const callTranscript = transcriptData.callTranscripts?.find( + (entry) => entry.callId === externalId + ) + const monologues = callTranscript?.transcript ?? [] - return { - externalId: metaData.id ?? externalId, - title: buildCallTitle(metaData), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: metaData.url || undefined, - contentHash: buildContentHash(metaData.id ?? externalId, metaData.started), - metadata: buildMetadata(metaData, participants), - } - } catch (error) { - logger.warn('Failed to get Gong call transcript', { - externalId, - error: toError(error).message, - }) + const hasSpokenText = monologues.some((monologue) => + (monologue.sentences ?? []).some((sentence) => Boolean(sentence.text?.trim())) + ) + if (!hasSpokenText) { + logger.info('Transcript not available for Gong call', { externalId }) return null } + + return { + externalId, + title: header.title, + content: formatTranscriptContent(header, monologues), + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: header.sourceUrl, + contentHash: header.contentHash, + metadata: header.metadata, + } }, validateConfig: async ( @@ -478,8 +556,23 @@ export const gongConnector: ConnectorConfig = { } try { + /** + * Probes the calls listing — the exact resource the sync reads — over a + * one-hour window so the check stays cheap. Probing `/v2/users` instead + * would pass for a key that lacks call-read access and only fail later, + * mid-sync. + */ + const toDateTime = new Date() + const fromDateTime = new Date(toDateTime.getTime() - 60 * 60 * 1000) + const query = new URLSearchParams({ + fromDateTime: fromDateTime.toISOString(), + toDateTime: toDateTime.toISOString(), + }) + const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim() + if (workspaceId) query.set('workspaceId', workspaceId) + const response = await fetchWithRetry( - `${GONG_API_BASE}/users`, + `${GONG_API_BASE}/calls?${query.toString()}`, { method: 'GET', headers: buildHeaders(accessToken), @@ -487,6 +580,14 @@ export const gongConnector: ConnectorConfig = { VALIDATE_RETRY_OPTIONS ) + /** + * `GET /v2/calls` documents `404 No calls found for the specified period` + * for a range that matches nothing. Most workspaces record nothing in any + * given hour, so a 404 here proves the credential authenticated and was + * allowed to query calls — a missing scope returns 401/403 instead. + */ + if (response.status === 404) return { valid: true } + if (!response.ok) { const errorText = await response.text().catch(() => '') return { @@ -509,12 +610,8 @@ export const gongConnector: ConnectorConfig = { result.callTitle = metadata.callTitle } - const participants = Array.isArray(metadata.participants) - ? (metadata.participants as string[]) - : [] - if (participants.length > 0) { - result.participants = participants.join(', ') - } + const participants = joinTagArray(metadata.participants) + if (participants) result.participants = participants if (metadata.duration != null) { const num = Number(metadata.duration) diff --git a/apps/sim/connectors/google-calendar/google-calendar.test.ts b/apps/sim/connectors/google-calendar/google-calendar.test.ts new file mode 100644 index 00000000000..00964259747 --- /dev/null +++ b/apps/sim/connectors/google-calendar/google-calendar.test.ts @@ -0,0 +1,147 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar' +import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' + +const ORGANIZER_EMAIL = 'organizer@example.com' +const ATTENDEE_EMAIL = 'attendee@example.com' +const ATTENDEE_NAME = 'Ada Lovelace' + +const EVENT = { + id: 'evt-1', + status: 'confirmed', + summary: 'Quarterly sync', + description: 'Agenda attached', + location: 'Room 4', + htmlLink: 'https://calendar.google.com/event?eid=evt-1', + created: '2026-01-01T00:00:00Z', + updated: '2026-01-02T00:00:00Z', + start: { dateTime: '2026-02-01T10:00:00Z', timeZone: 'UTC' }, + end: { dateTime: '2026-02-01T11:00:00Z', timeZone: 'UTC' }, + organizer: { email: ORGANIZER_EMAIL, displayName: 'Grace Hopper' }, + attendees: [ + { email: ATTENDEE_EMAIL, displayName: ATTENDEE_NAME }, + { email: 'second@example.com' }, + { email: 'room@resource.calendar.google.com', resource: true }, + ], +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + +beforeEach(() => { + fetchMock.mockReset() + fetchMock.mockImplementation(async (input) => { + const url = String(input) + if (url.includes('/events?')) return jsonResponse({ items: [EVENT] }) + if (url.includes('/events/')) return jsonResponse(EVENT) + throw new Error(`Unexpected fetch: ${url}`) + }) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +async function listOne(sourceConfig: Record) { + const result = await googleCalendarConnector.listDocuments('token', sourceConfig) + expect(result.documents).toHaveLength(1) + return result.documents[0] +} + +describe('google-calendar attendee PII opt-out', () => { + it('exposes an includeAttendees config field defaulting to on', () => { + const field = googleCalendarConnectorMeta.configFields.find((f) => f.id === 'includeAttendees') + expect(field).toBeDefined() + expect(field?.options?.map((o) => o.id)).toEqual(['true', 'false']) + expect(field?.description).toBeTruthy() + }) + + it('keeps the organizer tag definition even though the tag can now be absent', () => { + expect(googleCalendarConnectorMeta.tagDefinitions?.map((t) => t.id)).toContain('organizer') + expect(googleCalendarConnectorMeta.tagDefinitions?.map((t) => t.id)).toContain('attendeeCount') + }) + + it('indexes attendee and organizer identifiers when unset (default on)', async () => { + const doc = await listOne({}) + expect(doc.content).toContain(`Organizer: Grace Hopper (${ORGANIZER_EMAIL})`) + expect(doc.content).toContain(`Attendees: ${ATTENDEE_NAME}, second@example.com`) + expect(doc.contentHash).toBe('gcal:evt-1:2026-01-02T00:00:00Z') + + const tags = googleCalendarConnector.mapTags?.(doc.metadata ?? {}) ?? {} + expect(tags.organizer).toBe(`Grace Hopper (${ORGANIZER_EMAIL})`) + expect(tags.attendeeCount).toBe(2) + }) + + it('produces identical output when explicitly enabled', async () => { + const unset = await listOne({}) + const enabled = await listOne({ includeAttendees: 'true' }) + expect(enabled.content).toBe(unset.content) + expect(enabled.contentHash).toBe(unset.contentHash) + }) + + it('omits every attendee/organizer identifier from content and tags when off', async () => { + const doc = await listOne({ includeAttendees: 'false' }) + + const serialized = JSON.stringify(doc) + for (const identifier of [ + ORGANIZER_EMAIL, + ATTENDEE_EMAIL, + ATTENDEE_NAME, + 'second@example.com', + 'Grace Hopper', + ]) { + expect(serialized).not.toContain(identifier) + } + expect(doc.content).not.toContain('Organizer:') + expect(doc.content).not.toContain('@') + + expect(doc.content).toContain('Attendees: 2') + expect(doc.content).toContain('Event: Quarterly sync') + + const tags = googleCalendarConnector.mapTags?.(doc.metadata ?? {}) ?? {} + expect(tags.organizer).toBeUndefined() + expect(tags.attendeeCount).toBe(2) + expect(JSON.stringify(tags)).not.toContain('@') + }) + + it('changes the content hash when the toggle flips so existing documents re-hydrate', async () => { + const on = await listOne({}) + const off = await listOne({ includeAttendees: 'false' }) + expect(off.contentHash).not.toBe(on.contentHash) + }) + + it('keeps listDocuments and getDocument hashes byte-identical in both states', async () => { + for (const sourceConfig of [{}, { includeAttendees: 'false' }]) { + const listed = await listOne(sourceConfig) + const fetched = await googleCalendarConnector.getDocument('token', sourceConfig, 'evt-1') + expect(fetched).not.toBeNull() + expect(fetched?.contentHash).toBe(listed.contentHash) + expect(fetched?.content).toBe(listed.content) + } + }) + + it('keeps the multi-calendar hash namespaced and discriminated', async () => { + const config = { calendarId: ['a@group.calendar.google.com', 'b@group.calendar.google.com'] } + const on = await listOne(config) + const off = await listOne({ ...config, includeAttendees: 'false' }) + expect(on.contentHash).toBe('gcal:a@group.calendar.google.com:evt-1:2026-01-02T00:00:00Z') + expect(off.contentHash).toBe(`${on.contentHash}:noattendees`) + + const fetched = await googleCalendarConnector.getDocument( + 'token', + { ...config, includeAttendees: 'false' }, + 'a@group.calendar.google.com:evt-1' + ) + expect(fetched?.contentHash).toBe(off.contentHash) + }) +}) diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 90ed26a6b8c..921907a82ec 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -52,7 +52,7 @@ function formatEventTime(eventTime?: CalendarEventTime): string { if (!eventTime) return 'Unknown' if (eventTime.dateTime) { const date = new Date(eventTime.dateTime) - return date.toLocaleString('en-US', { + const options: Intl.DateTimeFormatOptions = { weekday: 'long', year: 'numeric', month: 'long', @@ -60,8 +60,23 @@ function formatEventTime(eventTime?: CalendarEventTime): string { hour: 'numeric', minute: '2-digit', timeZoneName: 'short', - timeZone: eventTime.timeZone || undefined, - }) + } + /** + * `start.timeZone` is a free-form IANA string echoed from the event. An + * unrecognized or legacy zone makes `toLocaleString` throw a RangeError, + * which would abort the whole listing page, so fall back to the runtime + * zone rather than failing the sync over one event. + */ + if (eventTime.timeZone) { + try { + return date.toLocaleString('en-US', { ...options, timeZone: eventTime.timeZone }) + } catch { + logger.warn('Unrecognized event time zone, formatting in runtime zone', { + timeZone: eventTime.timeZone, + }) + } + } + return date.toLocaleString('en-US', options) } if (eventTime.date) { const date = new Date(`${eventTime.date}T00:00:00`) @@ -82,6 +97,33 @@ function isAllDayEvent(event: CalendarEvent): boolean { return Boolean(event.start?.date && !event.start?.dateTime) } +/** + * Whether attendee/organizer identifiers may be indexed. The dropdown stores the + * string `'false'` to opt out; anything else — including an unset field on a source + * configured before the option existed — keeps identifiers. + */ +function readIncludeAttendees(sourceConfig: Record): boolean { + const value = sourceConfig.includeAttendees + return value !== 'false' && value !== false +} + +/** + * Discriminator appended to the metadata-only content hash when attendee + * identifiers are suppressed. Without it, flipping the toggle would leave every + * already-synced event hash-unchanged and the setting would never take effect on + * existing documents. The ON form stays byte-identical to the historical hash so + * turning the feature on (or leaving it unset) causes zero re-index churn. + */ +const NO_ATTENDEES_HASH_SUFFIX = ':noattendees' + +/** + * Counts attendees excluding rooms/equipment, matching what the content renderer lists. + */ +function countAttendees(attendees?: CalendarAttendee[]): number { + if (!attendees) return 0 + return attendees.filter((a) => !a.resource).length +} + /** * Formats attendees into a comma-separated list of names/emails. */ @@ -106,8 +148,13 @@ function formatOrganizer(organizer?: { email?: string; displayName?: string }): /** * Builds a readable content string from a calendar event. + * + * When `includeAttendees` is false the organizer line is dropped entirely and the + * attendee line degrades to a bare count: a count carries no identity, is already + * published as the `attendeeCount` tag, and keeps "how big was this meeting" + * answerable without naming anyone. */ -function eventToContent(event: CalendarEvent): string { +function eventToContent(event: CalendarEvent, includeAttendees: boolean): string { const parts: string[] = [] parts.push(`Event: ${event.summary || 'Untitled Event'}`) @@ -122,14 +169,21 @@ function eventToContent(event: CalendarEvent): string { parts.push(`Location: ${event.location}`) } - const organizer = formatOrganizer(event.organizer) - if (organizer) { - parts.push(`Organizer: ${organizer}`) - } + if (includeAttendees) { + const organizer = formatOrganizer(event.organizer) + if (organizer) { + parts.push(`Organizer: ${organizer}`) + } - const attendees = formatAttendees(event.attendees) - if (attendees) { - parts.push(`Attendees: ${attendees}`) + const attendees = formatAttendees(event.attendees) + if (attendees) { + parts.push(`Attendees: ${attendees}`) + } + } else { + const attendeeCount = countAttendees(event.attendees) + if (attendeeCount > 0) { + parts.push(`Attendees: ${attendeeCount}`) + } } if (event.description) { @@ -205,20 +259,22 @@ function getTimeRange(sourceConfig: Record): { timeMin: string; function eventToDocument( event: CalendarEvent, calendarId: string, - isMultiCalendar: boolean + isMultiCalendar: boolean, + includeAttendees: boolean ): ExternalDocument | null { if (event.status === 'cancelled') return null - const content = eventToContent(event) + const content = eventToContent(event, includeAttendees) if (!content.trim()) return null const startTime = event.start?.dateTime || event.start?.date || '' - const attendeeCount = event.attendees?.filter((a) => !a.resource).length || 0 + const attendeeCount = countAttendees(event.attendees) const externalId = isMultiCalendar ? `${calendarId}:${event.id}` : event.id - const contentHash = isMultiCalendar + const baseHash = isMultiCalendar ? `gcal:${calendarId}:${event.id}:${event.updated ?? ''}` : `gcal:${event.id}:${event.updated ?? ''}` + const contentHash = includeAttendees ? baseHash : `${baseHash}${NO_ATTENDEES_HASH_SUFFIX}` return { externalId, @@ -232,7 +288,7 @@ function eventToDocument( startTime, endTime: event.end?.dateTime || event.end?.date || '', location: event.location || '', - organizer: formatOrganizer(event.organizer), + organizer: includeAttendees ? formatOrganizer(event.organizer) : '', attendeeCount, isAllDay: isAllDayEvent(event), eventDate: startTime, @@ -284,10 +340,25 @@ export const googleCalendarConnector: ConnectorConfig = { const calendarId = calendarIds[calendarIndex] + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const rawMaxEvents = sourceConfig.maxEvents + ? Number(sourceConfig.maxEvents) + : DEFAULT_MAX_EVENTS + const maxEvents = Number.isFinite(rawMaxEvents) ? rawMaxEvents : 0 + const isCapped = maxEvents > 0 + /** + * Last-page precision: never ask Google for more events than the remaining + * cap allowance. `maxResults` is capped at 2500 by the API; PAGE_SIZE stays + * within that. + */ + const pageSize = isCapped + ? Math.max(1, Math.min(PAGE_SIZE, maxEvents - prevFetched)) + : PAGE_SIZE + const queryParams = new URLSearchParams({ singleEvents: 'true', orderBy: 'startTime', - maxResults: String(PAGE_SIZE), + maxResults: String(pageSize), timeMin, timeMax, }) @@ -333,19 +404,39 @@ export const googleCalendarConnector: ConnectorConfig = { const events = (data.items || []) as CalendarEvent[] const isMultiCalendar = calendarIds.length > 1 - const documents: ExternalDocument[] = [] + const includeAttendees = readIncludeAttendees(sourceConfig) + const allDocuments: ExternalDocument[] = [] for (const event of events) { - const doc = eventToDocument(event, calendarId, isMultiCalendar) - if (doc) documents.push(doc) + const doc = eventToDocument(event, calendarId, isMultiCalendar, includeAttendees) + if (doc) allDocuments.push(doc) } - const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length + let documents = allDocuments + if (isCapped) { + const remaining = Math.max(0, maxEvents - prevFetched) + if (allDocuments.length > remaining) documents = allDocuments.slice(0, remaining) + } + + const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched - const maxEvents = sourceConfig.maxEvents ? Number(sourceConfig.maxEvents) : DEFAULT_MAX_EVENTS - const hitLimit = maxEvents > 0 && totalFetched >= maxEvents + const nextPageToken = (data.nextPageToken as string | undefined) || undefined + const hasMoreCalendars = calendarIndex + 1 < calendarIds.length + const hitLimit = isCapped && totalFetched >= maxEvents - const nextPageToken = data.nextPageToken as string | undefined + /** + * `listingCapped` suppresses the sync engine's deletion reconciliation, so + * it is set only when the `maxEvents` cap actually truncated a larger + * source — events were dropped from this page, another page remains, or a + * configured calendar is still unwalked. A cap reached exactly at source + * exhaustion leaves it unset so events deleted upstream still reconcile. + * The `timeMin`/`timeMax` window is an intentional scope filter, never a + * cap, and is deliberately not flagged. + */ + const truncatedByCap = + hitLimit && + (documents.length < allDocuments.length || Boolean(nextPageToken) || hasMoreCalendars) + if (truncatedByCap && syncContext) syncContext.listingCapped = true if (hitLimit) { return { documents, hasMore: false } @@ -359,11 +450,10 @@ export const googleCalendarConnector: ConnectorConfig = { } } - const nextCalendarIndex = calendarIndex + 1 - if (nextCalendarIndex < calendarIds.length) { + if (hasMoreCalendars) { return { documents, - nextCursor: JSON.stringify({ calendarIndex: nextCalendarIndex }), + nextCursor: JSON.stringify({ calendarIndex: calendarIndex + 1 }), hasMore: true, } } @@ -429,7 +519,7 @@ export const googleCalendarConnector: ConnectorConfig = { if (event.status === 'cancelled') return null - return eventToDocument(event, calendarId, isMultiCalendar) ?? null + return eventToDocument(event, calendarId, isMultiCalendar, readIncludeAttendees(sourceConfig)) }, validateConfig: async ( diff --git a/apps/sim/connectors/google-calendar/meta.ts b/apps/sim/connectors/google-calendar/meta.ts index d2fcc3c5a87..dda94336817 100644 --- a/apps/sim/connectors/google-calendar/meta.ts +++ b/apps/sim/connectors/google-calendar/meta.ts @@ -59,7 +59,20 @@ export const googleCalendarConnectorMeta: ConnectorMeta = { type: 'short-input', placeholder: 'e.g. standup, sprint review (optional)', required: false, - description: 'Filter events by text search across all fields.', + description: + 'Free-text search. Google matches it against the event summary, description, location, and the organizer and attendee names and email addresses.', + }, + { + id: 'includeAttendees', + title: 'Include Attendees', + type: 'dropdown', + required: false, + options: [ + { label: 'Yes (default)', id: 'true' }, + { label: 'No', id: 'false' }, + ], + description: + 'When Yes, organizer and attendee names and email addresses are written into the indexed event text and into the Organizer tag. Indexed text is embedded into searchable chunks, so anyone with access to this knowledge base can retrieve those addresses — a wider audience than the calendar itself grants. Choose No to index a non-identifying attendee count instead and drop the Organizer tag.', }, { id: 'maxEvents', diff --git a/apps/sim/connectors/google-docs/google-docs.ts b/apps/sim/connectors/google-docs/google-docs.ts index f08b405eb52..fae56ea5f1a 100644 --- a/apps/sim/connectors/google-docs/google-docs.ts +++ b/apps/sim/connectors/google-docs/google-docs.ts @@ -5,13 +5,32 @@ import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { buildDriveParentsClause, + ConnectorFileTooLargeError, joinTagArray, + markSkipped, parseMultiValue, parseTagDate, + readBodyWithLimit, + sizeLimitSkipReason, } from '@/connectors/utils' const logger = createLogger('GoogleDocsConnector') +/** + * Ceiling for the raw `documents.get` JSON body, applied as a streaming memory + * guard. A Google Doc holds at most 1.02 million characters and the structured + * response inflates that with per-run styling, so this is an absolute bound + * rather than a multiple of `CONNECTOR_MAX_FILE_BYTES` — that cap is 100MB, and + * a multiple of it would both admit hundreds of megabytes into memory and sit so + * far above any reachable document that the guard could never fire. + */ +const MAX_DOCS_RESPONSE_BYTES = 100 * 1024 * 1024 + +/** Drive `files.list` page size. The API caps `pageSize` at 1000. */ +const PAGE_SIZE = 100 + +const GOOGLE_DOC_MIME_TYPE = 'application/vnd.google-apps.document' + /** * Represents a Google Drive file entry returned by the Drive API. */ @@ -26,30 +45,63 @@ interface DriveFile { } /** - * Represents a structural element within a Google Docs document body. + * A single element inside a paragraph. Only the variants that carry readable + * text are modeled — `pageBreak`, `columnBreak`, `horizontalRule`, `equation`, + * and `inlineObjectElement` contribute nothing to a plain-text rendering. + */ +interface DocsParagraphElement { + textRun?: { content?: string } + richLink?: { richLinkProperties?: { title?: string; uri?: string } } + person?: { personProperties?: { name?: string; email?: string } } +} + +/** + * Represents a structural element within a Google Docs document body. The Docs + * API models `body.content` as a union of `paragraph`, `table`, + * `tableOfContents`, and `sectionBreak`; table cells and the table of contents + * nest further `StructuralElement` arrays. */ interface DocsStructuralElement { paragraph?: { paragraphStyle?: { namedStyleType?: string } - elements?: { - textRun?: { - content?: string - } + bullet?: Record + elements?: DocsParagraphElement[] + } + table?: { + tableRows?: { + tableCells?: { + content?: DocsStructuralElement[] + }[] }[] } + tableOfContents?: { + content?: DocsStructuralElement[] + } } /** - * Represents the response from the Google Docs API for a single document. + * A tab of a Google Doc. Tabs may nest arbitrarily deep via `childTabs`. + */ +interface DocsTab { + documentTab?: { + body?: { content?: DocsStructuralElement[] } + } + childTabs?: DocsTab[] +} + +/** + * Represents the response from the Google Docs API for a single document. With + * `includeTabsContent=true` the content lands in `tabs` and the legacy `body` + * field is left empty; `body` is retained only as a fallback in case the request + * is ever served without tab content. */ interface DocsDocument { - documentId: string - title: string body?: { content?: DocsStructuralElement[] } + tabs?: DocsTab[] } /** @@ -75,44 +127,109 @@ function headingPrefix(namedStyleType?: string): string { } /** - * Extracts plain text from a Google Docs API structured document response. - * Headings are prefixed with Markdown-style `#` markers. + * Renders the readable text of a single paragraph element. `richLink` and + * `person` carry user-visible text that is absent from any `textRun`, so + * dropping them would silently lose linked-file titles and @-mentions. */ -function extractTextFromDocsBody(doc: DocsDocument): string { - const elements = doc.body?.content - if (!elements) return '' +function paragraphElementText(element: DocsParagraphElement): string { + if (element.textRun?.content) return element.textRun.content + if (element.richLink?.richLinkProperties) { + const { title, uri } = element.richLink.richLinkProperties + return title || uri || '' + } + if (element.person?.personProperties) { + const { name, email } = element.person.personProperties + return name || email || '' + } + return '' +} +/** + * Extracts plain text from a list of Docs structural elements, recursing into + * table cells and the table of contents. Headings are prefixed with + * Markdown-style `#` markers and list items with `- `. + */ +function extractTextFromStructuralElements(elements: DocsStructuralElement[]): string[] { const parts: string[] = [] for (const element of elements) { const paragraph = element.paragraph - if (!paragraph?.elements) continue + if (paragraph?.elements) { + const heading = headingPrefix(paragraph.paragraphStyle?.namedStyleType) + const prefix = heading || (paragraph.bullet ? '- ' : '') + /** + * Each paragraph's final `textRun.content` already ends with `\n`. Strip + * it before joining with `\n` so a heading followed by a body paragraph + * is separated by a single newline, not two. + */ + const text = paragraph.elements.map(paragraphElementText).join('').replace(/\n+$/, '') + + if (text.trim()) parts.push(`${prefix}${text}`) + continue + } - const prefix = headingPrefix(paragraph.paragraphStyle?.namedStyleType) - /** - * Each paragraph's final `textRun.content` already ends with `\n`. Strip - * it before joining with `\n` so a heading followed by a body paragraph - * is separated by a single newline, not two. - */ - const text = paragraph.elements - .map((el) => el.textRun?.content ?? '') - .join('') - .replace(/\n+$/, '') + if (element.table?.tableRows) { + for (const row of element.table.tableRows) { + for (const cell of row.tableCells ?? []) { + if (cell.content) parts.push(...extractTextFromStructuralElements(cell.content)) + } + } + continue + } - if (text.trim()) { - parts.push(`${prefix}${text}`) + if (element.tableOfContents?.content) { + parts.push(...extractTextFromStructuralElements(element.tableOfContents.content)) } } + return parts +} + +/** + * Collects the text of a tab and every descendant tab, depth-first in the order + * the Docs API returns them. + */ +function extractTextFromTabs(tabs: DocsTab[]): string[] { + const parts: string[] = [] + + for (const tab of tabs) { + const content = tab.documentTab?.body?.content + if (content) parts.push(...extractTextFromStructuralElements(content)) + if (tab.childTabs?.length) parts.push(...extractTextFromTabs(tab.childTabs)) + } + + return parts +} + +/** + * Extracts plain text from a Google Docs API document response. `tabs` is the + * source of truth because `includeTabsContent=true` moves all content there and + * leaves `body` empty; `body` is read only when `tabs` yields nothing, so a + * response served without tab content still indexes instead of coming back blank. + */ +function extractTextFromDocument(doc: DocsDocument): string { + const parts = doc.tabs?.length ? extractTextFromTabs(doc.tabs) : [] + if (parts.length === 0 && doc.body?.content) { + parts.push(...extractTextFromStructuralElements(doc.body.content)) + } + return parts.join('\n').trim() } /** - * Fetches the structured content of a Google Doc via the Docs API and - * extracts it as plain text. + * Fetches the structured content of a Google Doc via the Docs API and extracts + * it as plain text. `includeTabsContent=true` is required — without it the API + * returns only the first tab's content and every other tab is silently lost. + * Throws {@link ConnectorFileTooLargeError} when the response body exceeds + * {@link MAX_DOCS_RESPONSE_BYTES} so it surfaces as a visible skipped row rather + * than being buffered whole. */ async function fetchDocContent(accessToken: string, documentId: string): Promise { - const url = `https://docs.googleapis.com/v1/documents/${documentId}?fields=body.content` + const params = new URLSearchParams({ + includeTabsContent: 'true', + fields: 'body.content,tabs', + }) + const url = `https://docs.googleapis.com/v1/documents/${encodeURIComponent(documentId)}?${params.toString()}` const response = await fetchWithRetry(url, { method: 'GET', @@ -126,8 +243,11 @@ async function fetchDocContent(accessToken: string, documentId: string): Promise throw new Error(`Failed to fetch Google Doc content ${documentId}: ${response.status}`) } - const doc = (await response.json()) as DocsDocument - return extractTextFromDocsBody(doc) + const buffer = await readBodyWithLimit(response, MAX_DOCS_RESPONSE_BYTES) + if (!buffer) throw new ConnectorFileTooLargeError(MAX_DOCS_RESPONSE_BYTES) + + const doc = JSON.parse(buffer.toString('utf8')) as DocsDocument + return extractTextFromDocument(doc) } /** @@ -142,7 +262,15 @@ function fileToStub(file: DriveFile): ExternalDocument { contentDeferred: true, mimeType: 'text/plain', sourceUrl: file.webViewLink || `https://docs.google.com/document/d/${file.id}/edit`, - contentHash: `gdocs:${file.id}:${file.modifiedTime ?? ''}`, + /** + * The `v2` namespace is a one-time invalidation. The hash is metadata-only, + * so a stored document whose Drive `modifiedTime` has not moved is + * classified `unchanged` and never re-hydrated — it would keep content + * extracted before `includeTabsContent` and table traversal landed, i.e. + * missing every tab after the first and every table. Bumping the namespace + * forces one re-hydration per document, then normal hash gating resumes. + */ + contentHash: `gdocs:v2:${file.id}:${file.modifiedTime ?? ''}`, metadata: { modifiedTime: file.modifiedTime, createdTime: file.createdTime, @@ -155,7 +283,7 @@ function fileToStub(file: DriveFile): ExternalDocument { * Builds the Drive API query string for listing Google Docs. */ function buildQuery(sourceConfig: Record): string { - const parts: string[] = ['trashed = false', "mimeType = 'application/vnd.google-apps.document'"] + const parts: string[] = ['trashed = false', `mimeType = '${GOOGLE_DOC_MIME_TYPE}'`] const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId)) if (parentsClause) parts.push(parentsClause) @@ -173,12 +301,28 @@ export const googleDocsConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const query = buildQuery(sourceConfig) - const pageSize = 100 + const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0 + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = maxDocs > 0 ? maxDocs - previouslyFetched : 0 + const pageSize = remaining > 0 ? Math.min(PAGE_SIZE, remaining) : PAGE_SIZE + + /** + * `incompleteSearch` must be named in the partial-response mask — Drive's + * `fields` parameter filters the top-level response too, so omitting it + * leaves `data.incompleteSearch` permanently `undefined` and the + * reconciliation guard below dead. + * + * Drive returns items in an arbitrary order when `orderBy` is omitted, so a + * `maxDocs` cap would otherwise select a different, unpredictable subset on + * every sync. Sorting newest-first makes the capped scope deterministic. + */ const queryParams = new URLSearchParams({ q: query, pageSize: String(pageSize), - fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)', + orderBy: 'modifiedTime desc', + fields: + 'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)', supportsAllDrives: 'true', includeItemsFromAllDrives: 'true', }) @@ -219,17 +363,11 @@ export const googleDocsConnector: ConnectorConfig = { */ const incompleteSearch = data.incompleteSearch === true - const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0 - const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 - let documents = files.map(fileToStub) let slicedSome = false - if (maxDocs > 0) { - const remaining = maxDocs - previouslyFetched - if (documents.length > remaining) { - slicedSome = true - documents = documents.slice(0, remaining) - } + if (maxDocs > 0 && documents.length > remaining) { + slicedSome = true + documents = documents.slice(0, remaining) } const totalFetched = previouslyFetched + documents.length @@ -264,7 +402,7 @@ export const googleDocsConnector: ConnectorConfig = { externalId: string ): Promise => { const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed' - const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` + const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` const response = await fetchWithRetry(url, { method: 'GET', @@ -282,7 +420,7 @@ export const googleDocsConnector: ConnectorConfig = { const file = (await response.json()) as DriveFile & { trashed?: boolean } if (file.trashed) return null - if (file.mimeType !== 'application/vnd.google-apps.document') return null + if (file.mimeType !== GOOGLE_DOC_MIME_TYPE) return null try { const content = await fetchDocContent(accessToken, file.id) @@ -290,10 +428,21 @@ export const googleDocsConnector: ConnectorConfig = { return { ...fileToStub(file), content, contentDeferred: false } } catch (error) { + /** + * An oversized doc is a permanent, explainable outcome — surface it as a + * visible skipped (failed) row rather than dropping it silently. + */ + if (error instanceof ConnectorFileTooLargeError) { + return markSkipped(fileToStub(file), sizeLimitSkipReason(MAX_DOCS_RESPONSE_BYTES)) + } + /** + * Any other export failure is transient and must propagate so the engine records a + * failed hydration instead of silently dropping a document that still exists. + */ logger.warn(`Failed to extract content from document: ${file.name} (${file.id})`, { error: toError(error).message, }) - return null + throw toError(error) } }, @@ -343,8 +492,14 @@ export const googleDocsConnector: ConnectorConfig = { } } } else { - const url = - "https://www.googleapis.com/drive/v3/files?pageSize=1&q=mimeType%3D'application%2Fvnd.google-apps.document'&fields=files(id)" + const probeParams = new URLSearchParams({ + pageSize: '1', + q: `trashed = false and mimeType = '${GOOGLE_DOC_MIME_TYPE}'`, + fields: 'files(id)', + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + }) + const url = `https://www.googleapis.com/drive/v3/files?${probeParams.toString()}` const response = await fetchWithRetry( url, { diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index e7c2def3b5c..513aafebeec 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -14,6 +14,7 @@ import { parseTagDate, readBodyWithLimit, sizeLimitSkipReason, + stubOrSkipBySize, } from '@/connectors/utils' const logger = createLogger('GoogleDriveConnector') @@ -55,7 +56,7 @@ async function exportGoogleWorkspaceFile( throw new Error(`Unsupported Google Workspace MIME type: ${sourceMimeType}`) } - const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}` + const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=${encodeURIComponent(exportMimeType)}` const response = await fetchWithRetry(url, { method: 'GET', @@ -82,7 +83,10 @@ async function exportGoogleWorkspaceFile( } async function downloadTextFile(accessToken: string, fileId: string): Promise { - const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media` + // Listing runs with `includeItemsFromAllDrives`, so ids here can belong to a shared + // drive; `supportsAllDrives` declares that support to `files.get` the same way the + // metadata fetch in getDocument already does. (`files.export` takes no such param.) + const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&supportsAllDrives=true` const response = await fetchWithRetry(url, { method: 'GET', @@ -129,7 +133,6 @@ interface DriveFile { modifiedTime?: string createdTime?: string webViewLink?: string - parents?: string[] owners?: { displayName?: string; emailAddress?: string }[] size?: string starred?: boolean @@ -217,7 +220,7 @@ export const googleDriveConnector: ConnectorConfig = { pageSize: String(effectivePageSize), orderBy: 'modifiedTime desc', fields: - 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred)', + 'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred)', supportsAllDrives: 'true', includeItemsFromAllDrives: 'true', }) @@ -260,15 +263,26 @@ export const googleDriveConnector: ConnectorConfig = { const documents = files .filter((f) => isGoogleWorkspaceFile(f.mimeType) || isSupportedTextFile(f.mimeType)) - .map(fileToStub) + .map((f) => + stubOrSkipBySize(fileToStub(f), Number(f.size) || undefined, CONNECTOR_MAX_FILE_BYTES) + ) const totalFetched = previouslyFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxFiles > 0 && totalFetched >= maxFiles - if (syncContext && (hitLimit || incompleteSearch)) syncContext.listingCapped = true const nextPageToken = data.nextPageToken as string | undefined + /** + * Suppress deletion reconciliation only when the listing really is partial. + * Drive omits `nextPageToken` once the end of the list is reached, so hitting + * `maxFiles` on the final page still represents the full source set and must + * stay reconcilable — otherwise a capped source can never drop deleted files. + */ + if (syncContext && ((hitLimit && Boolean(nextPageToken)) || incompleteSearch)) { + syncContext.listingCapped = true + } + return { documents, nextCursor: hitLimit ? undefined : nextPageToken, @@ -282,8 +296,8 @@ export const googleDriveConnector: ConnectorConfig = { externalId: string ): Promise => { const fields = - 'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed' - const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` + 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed' + const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` const response = await fetchWithRetry(url, { method: 'GET', @@ -302,6 +316,19 @@ export const googleDriveConnector: ConnectorConfig = { if (file.trashed) return null + /** + * Mirrors the listing filter: a file re-uploaded under an unextractable type between + * listing and hydration has no content, which is an absence rather than a fetch + * failure. Returning null keeps it from being retried as a failure every sync. + */ + if (!isGoogleWorkspaceFile(file.mimeType) && !isSupportedTextFile(file.mimeType)) { + logger.info('Google Drive file has no extractable text type', { + fileId: file.id, + mimeType: file.mimeType, + }) + return null + } + try { const content = await fetchFileContent(accessToken, file.id, file.mimeType) if (!content.trim()) return null @@ -313,10 +340,16 @@ export const googleDriveConnector: ConnectorConfig = { logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name }) return markSkipped(fileToStub(file), sizeLimitSkipReason(error.limitBytes)) } + /** + * The file exists but its content could not be read. Propagate so the engine + * records a visible failed hydration instead of silently leaving a listed file + * unindexed (or, on an update, counting a stale copy as unchanged). + */ + const err = toError(error) logger.warn(`Failed to fetch content for file: ${file.name} (${file.id})`, { - error: toError(error).message, + error: err.message, }) - return null + throw err } }, @@ -369,7 +402,8 @@ export const googleDriveConnector: ConnectorConfig = { } } else { // Verify basic Drive access by listing one file - const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)' + const url = + 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true' const response = await fetchWithRetry( url, { diff --git a/apps/sim/connectors/google-forms/google-forms.ts b/apps/sim/connectors/google-forms/google-forms.ts index 1849fd0b380..443b394443a 100644 --- a/apps/sim/connectors/google-forms/google-forms.ts +++ b/apps/sim/connectors/google-forms/google-forms.ts @@ -18,12 +18,17 @@ const FORM_MIME_TYPE = 'application/vnd.google-apps.form' const FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder' /** - * Drive API page size when listing forms. The Drive API caps pageSize at 100. + * Drive API page size when listing forms. Drive coerces anything above 1000, but + * 100 is used because every listed form costs one `forms.get` plus (when + * responses are indexed) a full `forms.responses.list` walk for its change + * indicator, so a smaller page keeps per-call Forms API volume bounded. */ const DRIVE_PAGE_SIZE = 100 /** - * Maximum responses returned per Forms API page (API caps and defaults to 5000). + * Responses requested per `forms.responses.list` page. 5000 is what the API + * returns when `pageSize` is unspecified or zero; it is sent explicitly rather + * than relying on that default. No hard maximum is documented. */ const RESPONSES_PAGE_SIZE = 5000 @@ -215,31 +220,39 @@ async function fetchFormStructure( } /** - * Result of fetching a form's responses: the collected responses (capped at - * `MAX_RESPONSES_PER_FORM` for rendering) plus the greatest submission timestamp - * across ALL response pages. + * Result of scanning a form's responses: the retained responses (at most + * `retain`, in listing order) plus the greatest submission timestamp across ALL + * response pages. * - * `latestSubmittedTime` is tracked separately from the capped `responses` so the - * content hash computed in getDocument stays identical to the one computed during - * listing, which scans the same full set via `fetchLatestResponseTime`. If it - * were derived from the capped slice alone, a form with more than - * `MAX_RESPONSES_PER_FORM` responses could hash differently between the two paths - * and re-sync on every run. + * `latestSubmittedTime` is deliberately independent of `retain` so the change + * indicator is identical whether listing scanned with `retain: 0` or + * `getDocument` scanned with the configured render cap. Deriving it from the + * retained slice alone would let a form with more responses than the cap hash + * differently on the two paths and re-sync on every run. */ -interface FetchedResponses { +interface ScannedResponses { responses: FormResponse[] latestSubmittedTime?: string } /** - * Fetches form responses, retaining up to `MAX_RESPONSES_PER_FORM` for rendering. - * Every page is scanned for the latest submission timestamp even after the - * render cap is reached — the Forms API does not guarantee response order, so - * the newest submission may sit on any page. `fetchLatestResponseTime` scans - * the same full set during listing, keeping the content hash identical across - * the listing and getDocument paths regardless of form size. + * Walks every page of a form's responses, retaining at most `retain` of them. + * + * Every page is scanned for the latest submission timestamp even once `retain` + * is satisfied — the Forms API does not guarantee response order, so the newest + * submission may sit on any page. Listing calls this with `retain: 0` purely for + * the change indicator; `getDocument` passes the configured render cap. + * + * Throws on a failed read rather than returning partial data: a swallowed error + * would poison the stub's content hash and re-process the form on every sync, + * while throwing routes into the per-form catch that sets `skippedOnError` → + * `listingCapped`. */ -async function fetchFormResponses(accessToken: string, formId: string): Promise { +async function scanFormResponses( + accessToken: string, + formId: string, + retain: number +): Promise { const collected: FormResponse[] = [] let latest = '' let pageToken: string | undefined @@ -268,7 +281,7 @@ async function fetchFormResponses(accessToken: string, formId: string): Promise< if (pageLatest && pageLatest > latest) latest = pageLatest for (const r of responses) { - if (collected.length >= MAX_RESPONSES_PER_FORM) break + if (collected.length >= retain) break collected.push(r) } @@ -278,52 +291,6 @@ async function fetchFormResponses(accessToken: string, formId: string): Promise< return { responses: collected, latestSubmittedTime: latest || undefined } } -/** - * Reads the latest response submission time for change detection without - * retaining responses. Scans every page — the Forms API does not guarantee - * response order, so the newest submission may sit on any page. Returns the - * greatest `lastSubmittedTime` (falling back to `createTime`), or undefined - * when there are none. Throws on a failed read so the caller skips the form - * for this run instead of computing a hash from incomplete data — a swallowed - * error would poison the stub's content hash and re-process the form on every - * sync, while throwing routes into the per-form catch that sets - * `skippedOnError` → `listingCapped`. - */ -async function fetchLatestResponseTime( - accessToken: string, - formId: string -): Promise { - let latest = '' - let pageToken: string | undefined - - do { - const url = new URL(`${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}/responses`) - url.searchParams.set('pageSize', String(RESPONSES_PAGE_SIZE)) - if (pageToken) url.searchParams.set('pageToken', pageToken) - - const response = await fetchWithRetry(url.toString(), { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - throw new Error( - `Failed to read responses for change detection on form ${formId}: ${response.status}` - ) - } - - const data = (await response.json()) as FormResponseList - const pageLatest = latestResponseTime(data.responses ?? []) - if (pageLatest && pageLatest > latest) latest = pageLatest - pageToken = data.nextPageToken - } while (pageToken) - - return latest || undefined -} - /** * Returns the greatest submission timestamp across the given responses, or * undefined when the list is empty. @@ -487,7 +454,8 @@ export const googleFormsConnector: ConnectorConfig = { q: buildDriveQuery(folderIds), pageSize: String(DRIVE_PAGE_SIZE), orderBy: 'modifiedTime desc', - fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)', + fields: + 'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)', supportsAllDrives: 'true', includeItemsFromAllDrives: 'true', }) @@ -547,9 +515,23 @@ export const googleFormsConnector: ConnectorConfig = { const stubs = await mapWithConcurrency(files, LIST_CONCURRENCY, async (file) => { try { const form = await fetchFormStructure(accessToken, file.id) - if (!form) return null + if (!form) { + /** + * Drive listed this file moments ago, and Drive `q` already excluded + * trashed files, so the Forms API 404 is far more likely to be a + * transient read (permission propagation, eventual consistency) than a + * deletion — and the API gives no way to tell the two apart. Treating + * it as a truncated listing costs one deferred reconciliation; treating + * it as absence would hard-delete a live document. + */ + skippedOnError = true + logger.warn(`Form not readable via Forms API during listing: ${file.name} (${file.id})`) + return null + } const latest = - contentScope === 'both' ? await fetchLatestResponseTime(accessToken, file.id) : undefined + contentScope === 'both' + ? (await scanFormResponses(accessToken, file.id, 0)).latestSubmittedTime + : undefined return formToStub({ file, formTitle: form.info?.title || form.info?.documentTitle, @@ -635,18 +617,24 @@ export const googleFormsConnector: ConnectorConfig = { try { const form = await fetchFormStructure(accessToken, file.id) - if (!form) return null + /** + * The Drive metadata read above already confirmed a live, untrashed form, so a + * Forms API 404 here is an inconsistent read (permission propagation, eventual + * consistency) rather than a deletion — the same conclusion `listDocuments` + * reaches. Genuine deletion is observed by the form leaving the Drive listing, + * which reconciliation handles; returning null here would hard-delete instead. + */ + if (!form) { + throw new Error(`Form ${file.id} is listed in Drive but not readable via the Forms API`) + } const responseCap = resolveResponseCap(sourceConfig) const fetched = contentScope === 'both' - ? await fetchFormResponses(accessToken, file.id) + ? await scanFormResponses(accessToken, file.id, responseCap) : { responses: [], latestSubmittedTime: undefined } - const responses = fetched.responses - const cappedResponses = - responses.length > responseCap ? responses.slice(0, responseCap) : responses - const content = renderFormDocument(form, cappedResponses) + const content = renderFormDocument(form, fetched.responses) if (!content.trim()) return null const stub = formToStub({ @@ -659,10 +647,15 @@ export const googleFormsConnector: ConnectorConfig = { }) return { ...stub, content, contentDeferred: false } } catch (error) { + /** + * Absence is already covered above (404 Drive metadata, trashed, non-form MIME + * type). Anything reaching here is transient and must propagate so the engine + * records a failed hydration instead of silently dropping a form that still exists. + */ logger.warn(`Failed to fetch content for form: ${file.name} (${file.id})`, { error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/google-meet/google-meet.test.ts b/apps/sim/connectors/google-meet/google-meet.test.ts new file mode 100644 index 00000000000..3baf293ceeb --- /dev/null +++ b/apps/sim/connectors/google-meet/google-meet.test.ts @@ -0,0 +1,166 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { googleMeetConnector } from '@/connectors/google-meet/google-meet' +import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta' + +const RECORD_NAME = 'conferenceRecords/abc123' +const SIGNED_IN_NAME = 'Ada Lovelace' +const ANONYMOUS_NAME = 'guest-from-acme' +const PHONE_NAME = '+1 (555) •••-1234' + +const RECORD = { + name: RECORD_NAME, + startTime: '2026-02-01T10:00:00Z', + endTime: '2026-02-01T10:30:00Z', +} + +const PARTICIPANTS = [ + { name: `${RECORD_NAME}/participants/p1`, signedinUser: { displayName: SIGNED_IN_NAME } }, + { name: `${RECORD_NAME}/participants/p2`, anonymousUser: { displayName: ANONYMOUS_NAME } }, + { name: `${RECORD_NAME}/participants/p3`, phoneUser: { displayName: PHONE_NAME } }, +] + +const ENTRIES = [ + { + name: 'e1', + participant: `${RECORD_NAME}/participants/p1`, + text: 'Welcome everyone', + startTime: '2026-02-01T10:01:00Z', + }, + { + name: 'e2', + participant: `${RECORD_NAME}/participants/p3`, + text: 'Dialing in from the road', + startTime: '2026-02-01T10:02:00Z', + }, + { + name: 'e3', + participant: `${RECORD_NAME}/participants/p1`, + text: 'Lets begin', + startTime: '2026-02-01T10:03:00Z', + }, +] + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + +beforeEach(() => { + fetchMock.mockReset() + fetchMock.mockImplementation(async (input) => { + const url = String(input) + if (url.includes('/entries')) return jsonResponse({ transcriptEntries: ENTRIES }) + if (url.includes('/participants')) return jsonResponse({ participants: PARTICIPANTS }) + if (url.includes('/transcripts')) { + return jsonResponse({ + transcripts: [ + { + name: `${RECORD_NAME}/transcripts/t1`, + state: 'FILE_GENERATED', + docsDestination: { document: 'doc-1' }, + }, + ], + }) + } + if (url.includes('/conferenceRecords?')) { + return jsonResponse({ conferenceRecords: [RECORD] }) + } + if (url.includes('/conferenceRecords/')) return jsonResponse(RECORD) + throw new Error(`Unexpected fetch: ${url}`) + }) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +async function listStub(sourceConfig: Record) { + const result = await googleMeetConnector.listDocuments('token', sourceConfig) + expect(result.documents).toHaveLength(1) + return result.documents[0] +} + +describe('google-meet participant PII opt-out', () => { + it('exposes an includeParticipants config field defaulting to on', () => { + const field = googleMeetConnectorMeta.configFields.find((f) => f.id === 'includeParticipants') + expect(field).toBeDefined() + expect(field?.options?.map((o) => o.id)).toEqual(['true', 'false']) + expect(field?.description).toBeTruthy() + }) + + it('keeps the participants tag definition even though the tag can now be absent', () => { + expect(googleMeetConnectorMeta.tagDefinitions?.map((t) => t.id)).toContain('participants') + }) + + it('indexes participant display names when unset (default on)', async () => { + const doc = await googleMeetConnector.getDocument('token', {}, RECORD_NAME) + expect(doc).not.toBeNull() + expect(doc?.content).toContain( + `Participants: ${SIGNED_IN_NAME}, ${ANONYMOUS_NAME}, ${PHONE_NAME}` + ) + expect(doc?.content).toContain(`${SIGNED_IN_NAME}: Welcome everyone`) + expect(doc?.content).toContain(`${PHONE_NAME}: Dialing in from the road`) + expect(doc?.contentHash).toBe('gmeet:conferenceRecords/abc123:2026-02-01T10:30:00Z') + + const tags = googleMeetConnector.mapTags?.(doc?.metadata ?? {}) ?? {} + expect(String(tags.participants)).toContain(SIGNED_IN_NAME) + expect(String(tags.participants)).toContain(PHONE_NAME) + }) + + it('produces identical output when explicitly enabled', async () => { + const unset = await googleMeetConnector.getDocument('token', {}, RECORD_NAME) + const enabled = await googleMeetConnector.getDocument( + 'token', + { includeParticipants: 'true' }, + RECORD_NAME + ) + expect(enabled?.content).toBe(unset?.content) + expect(enabled?.contentHash).toBe(unset?.contentHash) + }) + + it('omits names and phone fragments from content and tags when off', async () => { + const doc = await googleMeetConnector.getDocument( + 'token', + { includeParticipants: 'false' }, + RECORD_NAME + ) + expect(doc).not.toBeNull() + + const serialized = JSON.stringify(doc) + for (const identifier of [SIGNED_IN_NAME, ANONYMOUS_NAME, PHONE_NAME, '1234', '555']) { + expect(serialized).not.toContain(identifier) + } + + expect(doc?.content).toContain('Participants: 3') + expect(doc?.content).toContain('Speaker 1: Welcome everyone') + expect(doc?.content).toContain('Speaker 2: Dialing in from the road') + expect(doc?.content).toContain('Speaker 1: Lets begin') + + const tags = googleMeetConnector.mapTags?.(doc?.metadata ?? {}) ?? {} + expect(tags.participants).toBeUndefined() + expect(tags.duration).toBe(30) + }) + + it('changes the content hash when the toggle flips so existing documents re-hydrate', async () => { + const on = await listStub({}) + const off = await listStub({ includeParticipants: 'false' }) + expect(off.contentHash).toBe(`${on.contentHash}:noparticipants`) + }) + + it('keeps the listing stub and getDocument hashes byte-identical in both states', async () => { + for (const sourceConfig of [{}, { includeParticipants: 'false' }]) { + const stub = await listStub(sourceConfig) + const fetched = await googleMeetConnector.getDocument('token', sourceConfig, RECORD_NAME) + expect(fetched).not.toBeNull() + expect(fetched?.contentHash).toBe(stub.contentHash) + } + }) +}) diff --git a/apps/sim/connectors/google-meet/google-meet.ts b/apps/sim/connectors/google-meet/google-meet.ts index ab04f74eae5..5f7f2a3813d 100644 --- a/apps/sim/connectors/google-meet/google-meet.ts +++ b/apps/sim/connectors/google-meet/google-meet.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -15,8 +15,8 @@ const RECORDS_PAGE_SIZE = 100 const TRANSCRIPTS_PAGE_SIZE = 100 /** Transcript entries page size (Meet API max is 100). */ const ENTRIES_PAGE_SIZE = 100 -/** Max concurrent participant-name lookups during a single getDocument call. */ -const PARTICIPANT_FETCH_CONCURRENCY = 5 +/** Participants list page size (Meet API max is 250). */ +const PARTICIPANTS_PAGE_SIZE = 250 /** * A conference record as returned by the Meet REST API v2. A conference record @@ -92,6 +92,11 @@ interface Participant { phoneUser?: { displayName?: string } } +interface ParticipantsListResponse { + participants?: Participant[] + nextPageToken?: string +} + function meetHeaders(accessToken: string): Record { return { Authorization: `Bearer ${accessToken}` } } @@ -126,28 +131,50 @@ function recordDurationMinutes(record: ConferenceRecord): number | undefined { return Math.round((end - start) / 60000) } +/** + * Whether participant display names may be indexed. The dropdown stores the string + * `'false'` to opt out; anything else — including an unset field on a source + * configured before the option existed — keeps display names. + */ +function readIncludeParticipants(sourceConfig: Record): boolean { + const value = sourceConfig.includeParticipants + return value !== 'false' && value !== false +} + +/** + * Discriminator appended to the metadata-only content hash when participant + * identifiers are suppressed. Without it, flipping the toggle would leave every + * already-synced meeting hash-unchanged and the setting would never take effect on + * existing documents. The ON form stays byte-identical to the historical hash so + * turning the feature on (or leaving it unset) causes zero re-index churn. + */ +const NO_PARTICIPANTS_HASH_SUFFIX = ':noparticipants' + /** * Computes the metadata-based change-detection hash for a conference record. Records * are immutable once ended, so the end time fully captures the final state; an * in-progress meeting (no end time) re-syncs once it ends and the hash changes. The - * identical formula is used for both the listing stub and the fetched document. + * identical formula is used for both the listing stub and the fetched document, so the + * participant-privacy discriminator must be applied on both sides or every sync would + * see a hash mismatch and re-index. */ -function buildContentHash(record: ConferenceRecord): string { - return `gmeet:${record.name}:${record.endTime ?? ''}` +function buildContentHash(record: ConferenceRecord, includeParticipants: boolean): string { + const base = `gmeet:${record.name}:${record.endTime ?? ''}` + return includeParticipants ? base : `${base}${NO_PARTICIPANTS_HASH_SUFFIX}` } /** * Builds the deferred listing stub for a conference record. Transcript content is * fetched lazily in getDocument; only metadata and the change hash are computed here. */ -function recordToStub(record: ConferenceRecord): ExternalDocument { +function recordToStub(record: ConferenceRecord, includeParticipants: boolean): ExternalDocument { return { externalId: record.name, title: recordTitle(record), content: '', contentDeferred: true, mimeType: 'text/plain', - contentHash: buildContentHash(record), + contentHash: buildContentHash(record, includeParticipants), metadata: { meetingDate: record.startTime, duration: recordDurationMinutes(record), @@ -166,15 +193,16 @@ function entryStartMs(entry: TranscriptEntry): number { } /** - * Resolves a participant's display name across the identity oneof, falling back to a - * stable placeholder when no name is exposed (e.g. anonymous joins). + * Resolves a participant's display name across the identity oneof, or undefined when + * no name is exposed (e.g. an anonymous join that supplied none) so callers can fall + * back to a placeholder without polluting the participant list. */ -function participantDisplayName(participant: Participant): string { +function participantDisplayName(participant: Participant): string | undefined { return ( participant.signedinUser?.displayName?.trim() || participant.anonymousUser?.displayName?.trim() || participant.phoneUser?.displayName?.trim() || - 'Unknown' + undefined ) } @@ -248,75 +276,142 @@ async function fetchTranscriptEntries( return entries } +/** Trailing id segment of a resource name, used as a secondary lookup key. */ +function resourceIdSegment(resourceName: string): string { + return resourceName.slice(resourceName.lastIndexOf('/') + 1) +} + /** - * Resolves the display names for a set of participant resource names, returning a map - * keyed by resource name. Participants that fail to resolve are omitted so the caller - * falls back to a placeholder. + * Lists every participant of a conference in one paginated sweep and returns their + * display names. The map is keyed by both the full resource name and its trailing id + * segment: the Meet API documents `TranscriptEntry.participant` only as "Refers to the + * participant who speaks" without pinning down its format, so indexing both shapes + * keeps speaker attribution working either way. + * + * A failure throws rather than degrading: the content hash is keyed on the conference's + * (already fixed) end time, so a document persisted with its roster silently missing + * would never be re-hydrated. Failing the hydration lets the next sync retry it. */ -async function resolveParticipantNames( +async function fetchParticipants( accessToken: string, - participantNames: string[] -): Promise> { - const map = new Map() - for (let i = 0; i < participantNames.length; i += PARTICIPANT_FETCH_CONCURRENCY) { - const batch = participantNames.slice(i, i + PARTICIPANT_FETCH_CONCURRENCY) - await Promise.all( - batch.map(async (name) => { - try { - const response = await fetchWithRetry(`${MEET_API_BASE}/${name}`, { - method: 'GET', - headers: meetHeaders(accessToken), - }) - if (!response.ok) return - const participant = (await response.json()) as Participant - map.set(name, participantDisplayName(participant)) - } catch (error) { - logger.warn('Failed to resolve Google Meet participant', { - participant: name, - error: toError(error).message, - }) - } - }) + recordName: string +): Promise<{ + namesByKey: Map + displayNames: Set + participantCount: number +}> { + const namesByKey = new Map() + const displayNames = new Set() + let participantCount = 0 + + let pageToken: string | undefined + do { + const params = new URLSearchParams({ pageSize: String(PARTICIPANTS_PAGE_SIZE) }) + if (pageToken) params.set('pageToken', pageToken) + const response = await fetchWithRetry( + `${MEET_API_BASE}/${recordName}/participants?${params.toString()}`, + { method: 'GET', headers: meetHeaders(accessToken) } ) - } - return map + if (!response.ok) { + throw new Error(`Failed to list Google Meet participants: ${response.status}`) + } + const data = (await response.json()) as ParticipantsListResponse + for (const participant of data.participants ?? []) { + if (!participant.name) continue + participantCount++ + const displayName = participantDisplayName(participant) + if (!displayName) continue + namesByKey.set(participant.name, displayName) + namesByKey.set(resourceIdSegment(participant.name), displayName) + displayNames.add(displayName) + } + pageToken = data.nextPageToken + } while (pageToken) + + return { namesByKey, displayNames, participantCount } +} + +/** Resolves a transcript entry's speaker name, tolerating either participant-key shape. */ +function speakerFor(entry: TranscriptEntry, namesByKey: Map): string | undefined { + if (!entry.participant) return undefined + return namesByKey.get(entry.participant) ?? namesByKey.get(resourceIdSegment(entry.participant)) +} + +interface TranscriptContentOptions { + namesByKey: Map + displayNames: Set + participantCount: number + includeParticipants: boolean } /** * Formats a meeting header plus speaker-attributed transcript lines into plain text. + * + * When `includeParticipants` is false, the participant roster degrades to a bare count + * and every speaker label becomes a pseudonym (`Speaker 1`, `Speaker 2`, …) assigned in + * order of first utterance. A count and a per-document pseudonym carry no identity, but + * keep the transcript readable as a dialogue and keep "how many people were in this + * meeting" answerable — dropping attribution entirely would merge distinct speakers into + * one undifferentiated wall of text. */ function formatTranscriptContent( record: ConferenceRecord, entries: TranscriptEntry[], - participantNames: Map + options: TranscriptContentOptions ): string { + const { namesByKey, displayNames, participantCount, includeParticipants } = options const parts: string[] = [] parts.push(`Meeting: ${recordTitle(record)}`) if (record.startTime) parts.push(`Date: ${record.startTime}`) const minutes = recordDurationMinutes(record) if (minutes != null) parts.push(`Duration: ${minutes} minutes`) - - const speakers = Array.from( - new Set( - entries - .map((entry) => (entry.participant ? participantNames.get(entry.participant) : undefined)) - .filter((name): name is string => Boolean(name)) - ) - ) - if (speakers.length > 0) parts.push(`Participants: ${speakers.join(', ')}`) + if (includeParticipants) { + if (displayNames.size > 0) parts.push(`Participants: ${[...displayNames].join(', ')}`) + } else if (participantCount > 0) { + parts.push(`Participants: ${participantCount}`) + } parts.push('') parts.push('--- Transcript ---') + const pseudonyms = new Map() for (const entry of entries) { const text = entry.text?.trim() if (!text) continue - const speaker = (entry.participant && participantNames.get(entry.participant)) || 'Unknown' + let speaker: string + if (includeParticipants) { + speaker = speakerFor(entry, namesByKey) ?? 'Unknown' + } else if (entry.participant) { + const key = resourceIdSegment(entry.participant) + let pseudonym = pseudonyms.get(key) + if (!pseudonym) { + pseudonym = `Speaker ${pseudonyms.size + 1}` + pseudonyms.set(key, pseudonym) + } + speaker = pseudonym + } else { + speaker = 'Unknown' + } parts.push(`${speaker}: ${text}`) } return parts.join('\n') } +/** + * Browsable URL for a transcript's exported Google Doc. `DocsDestination.exportUri` is + * documented as "URI for the Google Docs transcript file", so it is used verbatim. The + * same field documents the fallback template used when only `document` is present: "Use + * `https://docs.google.com/document/d/{$DocumentId}/view` to browse the transcript in + * the browser." + */ +function transcriptSourceUrl(transcripts: Transcript[]): string | undefined { + const exportUri = transcripts.find((t) => t.docsDestination?.exportUri)?.docsDestination + ?.exportUri + if (exportUri) return exportUri + const documentId = transcripts.find((t) => t.docsDestination?.document)?.docsDestination?.document + return documentId ? `https://docs.google.com/document/d/${documentId}/view` : undefined +} + /** * Builds the conference records list `filter` from the connector's scoping config. * Only the documented `start_time` filter is emitted, and only when a lookback window @@ -373,9 +468,10 @@ export const googleMeetConnector: ConnectorConfig = { const records = data.conferenceRecords ?? [] const nextPageToken = data.nextPageToken?.trim() || undefined + const includeParticipants = readIncludeParticipants(sourceConfig) const allDocuments = records .filter((record) => Boolean(record.name)) - .map((record) => recordToStub(record)) + .map((record) => recordToStub(record, includeParticipants)) let documents = allDocuments if (maxMeetings > 0) { @@ -406,80 +502,66 @@ export const googleMeetConnector: ConnectorConfig = { getDocument: async ( accessToken: string, - _sourceConfig: Record, + sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null - const recordName = conferenceResourceName(externalId) - - const record = await fetchConferenceRecord(accessToken, recordName) - if (!record) return null - - const transcripts = await fetchTranscripts(accessToken, recordName) - if (transcripts.length === 0) return null - - // Only index once every transcript is fully generated. Before then the entry set - // is still being populated, and because the content hash is keyed on the (now - // fixed) conference endTime, a partial transcript stored here would never be - // refreshed on later syncs. Waiting for FILE_GENERATED keeps indexed content final. - if (transcripts.some((transcript) => transcript.state !== 'FILE_GENERATED')) { - logger.info('Google Meet transcript not finalized yet', { externalId }) - return null - } + if (!externalId) return null + const recordName = conferenceResourceName(externalId) - const entryGroups = await Promise.all( - transcripts.map((transcript) => fetchTranscriptEntries(accessToken, transcript.name)) - ) - // The API guarantees chronological order only within a single transcript, so sort - // the merged entries by start time to keep speaker lines in sequence when a - // conference has more than one transcript. - const entries = entryGroups.flat().sort((a, b) => entryStartMs(a) - entryStartMs(b)) - - const hasText = entries.some((entry) => entry.text?.trim()) - if (!hasText) { - logger.info('Transcript not yet available for Google Meet conference', { externalId }) - return null - } + const record = await fetchConferenceRecord(accessToken, recordName) + if (!record) return null - const participantNames = await resolveParticipantNames( - accessToken, - Array.from( - new Set( - entries - .map((entry) => entry.participant) - .filter((name): name is string => Boolean(name)) - ) - ) - ) + const transcripts = await fetchTranscripts(accessToken, recordName) + if (transcripts.length === 0) return null - const content = formatTranscriptContent(record, entries, participantNames) - const sourceUrl = transcripts.find((t) => t.docsDestination?.exportUri)?.docsDestination - ?.exportUri - - const speakers = Array.from(new Set(Array.from(participantNames.values()))) - - return { - externalId: record.name, - title: recordTitle(record), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: sourceUrl || undefined, - contentHash: buildContentHash(record), - metadata: { - meetingDate: record.startTime, - duration: recordDurationMinutes(record), - participants: speakers, - }, - } - } catch (error) { - logger.warn('Failed to get Google Meet transcript', { - externalId, - error: toError(error).message, - }) + // Only index once every transcript is fully generated. Before then the entry set + // is still being populated, and because the content hash is keyed on the (now + // fixed) conference endTime, a partial transcript stored here would never be + // refreshed on later syncs. Waiting for FILE_GENERATED keeps indexed content final. + if (transcripts.some((transcript) => transcript.state !== 'FILE_GENERATED')) { + logger.info('Google Meet transcript not finalized yet', { externalId }) + return null + } + + const entryGroups = await Promise.all( + transcripts.map((transcript) => fetchTranscriptEntries(accessToken, transcript.name)) + ) + // The API guarantees chronological order only within a single transcript, so sort + // the merged entries by start time to keep speaker lines in sequence when a + // conference has more than one transcript. + const entries = entryGroups.flat().sort((a, b) => entryStartMs(a) - entryStartMs(b)) + + const hasText = entries.some((entry) => entry.text?.trim()) + if (!hasText) { + logger.info('Transcript not yet available for Google Meet conference', { externalId }) return null } + + const includeParticipants = readIncludeParticipants(sourceConfig) + const { namesByKey, displayNames, participantCount } = await fetchParticipants( + accessToken, + recordName + ) + + return { + externalId: record.name, + title: recordTitle(record), + content: formatTranscriptContent(record, entries, { + namesByKey, + displayNames, + participantCount, + includeParticipants, + }), + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: transcriptSourceUrl(transcripts), + contentHash: buildContentHash(record, includeParticipants), + metadata: { + meetingDate: record.startTime, + duration: recordDurationMinutes(record), + participants: includeParticipants ? [...displayNames] : [], + }, + } }, validateConfig: async ( diff --git a/apps/sim/connectors/google-meet/meta.ts b/apps/sim/connectors/google-meet/meta.ts index b7da8758ca2..88112e590d4 100644 --- a/apps/sim/connectors/google-meet/meta.ts +++ b/apps/sim/connectors/google-meet/meta.ts @@ -29,8 +29,21 @@ export const googleMeetConnectorMeta: ConnectorMeta = { type: 'short-input', required: false, mode: 'advanced', - placeholder: 'e.g. 90 (default: all time)', - description: 'Only sync meetings from the last N days. Leave blank to sync any age.', + placeholder: 'e.g. 30 (default: all available)', + description: + 'Only sync meetings from the last N days. Google keeps transcript entry data for 30 days after a conference ends and deletes the conference record itself on the same schedule, so older meetings have nothing left to index.', + }, + { + id: 'includeParticipants', + title: 'Include Participants', + type: 'dropdown', + required: false, + options: [ + { label: 'Yes (default)', id: 'true' }, + { label: 'No', id: 'false' }, + ], + description: + 'When Yes, participant display names are written into the indexed transcript — in the participant list, and as the speaker label on every line — and into the Participants tag. Dial-in participants are named by a partially redacted phone number and anonymous joiners by whatever name they typed. Indexed text is embedded into searchable chunks, so anyone with access to this knowledge base can retrieve those identifiers. Choose No to index a participant count and pseudonymous speaker labels instead, and drop the Participants tag.', }, ], diff --git a/apps/sim/connectors/google-sheets/google-sheets.test.ts b/apps/sim/connectors/google-sheets/google-sheets.test.ts index 66eb84aa434..56c0d9b540c 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.test.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.test.ts @@ -73,10 +73,22 @@ const SPREADSHEET_METADATA = { ], } +/** Adds a chart tab and returns the tabs out of index order. */ +const SPREADSHEET_METADATA_WITH_OBJECT_SHEET = { + spreadsheetId: SPREADSHEET_ID, + properties: { title: 'Quarterly Plan' }, + sheets: [ + { properties: { sheetId: 7, title: 'Costs', index: 1, sheetType: 'GRID' } }, + { properties: { sheetId: 9, title: 'Chart', index: 2, sheetType: 'OBJECT' } }, + { properties: { sheetId: 0, title: "Ann's Revenue", index: 0, sheetType: 'GRID' } }, + ], +} + /** Drive response bodies keyed by the scenario each test exercises. */ interface FetchStubResponses { drive: { status: number; body: unknown } values?: unknown + spreadsheet?: unknown } /** @@ -97,7 +109,9 @@ function stubFetch(responses: FetchStubResponses) { return new Response(JSON.stringify(responses.values ?? {}), { status: 200 }) } if (url.startsWith('https://sheets.googleapis.com/v4/spreadsheets/')) { - return new Response(JSON.stringify(SPREADSHEET_METADATA), { status: 200 }) + return new Response(JSON.stringify(responses.spreadsheet ?? SPREADSHEET_METADATA), { + status: 200, + }) } throw new Error(`Unexpected fetch to ${url}`) }) @@ -215,6 +229,132 @@ describe('googleSheetsConnector trashed handling', () => { }) }) + describe('listDocuments sheet selection', () => { + it('drops object (chart) tabs and orders the rest by tab index', async () => { + stubFetch({ + drive: { status: 200, body: { trashed: false } }, + spreadsheet: SPREADSHEET_METADATA_WITH_OBJECT_SHEET, + }) + + const result = await googleSheetsConnector.listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + + expect(result.documents.map((d) => d.externalId)).toEqual([ + `${SPREADSHEET_ID}__sheet__0`, + `${SPREADSHEET_ID}__sheet__7`, + ]) + }) + + it('selects the leftmost grid tab for the first-sheet filter', async () => { + stubFetch({ + drive: { status: 200, body: { trashed: false } }, + spreadsheet: SPREADSHEET_METADATA_WITH_OBJECT_SHEET, + }) + + const result = await googleSheetsConnector.listDocuments(ACCESS_TOKEN, { + ...SOURCE_CONFIG, + sheetFilter: 'first', + }) + + expect(result.documents.map((d) => d.externalId)).toEqual([`${SPREADSHEET_ID}__sheet__0`]) + }) + + it('tolerates a metadata response without a sheets array', async () => { + stubFetch({ + drive: { status: 200, body: { trashed: false } }, + spreadsheet: { spreadsheetId: SPREADSHEET_ID, properties: { title: 'Empty' } }, + }) + + const result = await googleSheetsConnector.listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + + expect(result).toEqual({ documents: [], hasMore: false }) + }) + }) + + describe('content extraction', () => { + it('keeps the stub contentHash identical between listing and hydration', async () => { + stubFetch({ + drive: { status: 200, body: { modifiedTime: '2026-07-01T00:00:00.000Z' } }, + values: { values: [['Region'], ['West']] }, + }) + + const listed = await googleSheetsConnector.listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + const hydrated = await googleSheetsConnector.getDocument( + ACCESS_TOKEN, + SOURCE_CONFIG, + `${SPREADSHEET_ID}__sheet__0` + ) + + expect(hydrated?.contentHash).toBe(listed.documents[0].contentHash) + }) + + it('keeps columns whose header cell is blank, sizing by the widest row', async () => { + stubFetch({ + drive: { status: 200, body: { trashed: false } }, + values: { values: [['Region'], ['West', '10', '20']] }, + }) + + const doc = await googleSheetsConnector.getDocument( + ACCESS_TOKEN, + SOURCE_CONFIG, + `${SPREADSHEET_ID}__sheet__0` + ) + + expect(doc?.content).toContain('Region: West') + expect(doc?.content).toContain('Column 2: 10') + expect(doc?.content).toContain('Column 3: 20') + expect(doc?.metadata?.columnCount).toBe(3) + }) + + it('requests every column via a row-only A1 range with the tab name quote-escaped', async () => { + const fetchMock = stubFetch({ + drive: { status: 200, body: { trashed: false } }, + spreadsheet: SPREADSHEET_METADATA_WITH_OBJECT_SHEET, + values: { values: [['Region'], ['West']] }, + }) + + await googleSheetsConnector.getDocument( + ACCESS_TOKEN, + SOURCE_CONFIG, + `${SPREADSHEET_ID}__sheet__0` + ) + + const valuesUrl = fetchMock.mock.calls + .map(([input]) => String(input)) + .find((url) => url.includes('/values/')) + + expect(valuesUrl).toContain(encodeURIComponent("'Ann''s Revenue'!1:10000")) + expect(valuesUrl).not.toContain('ZZ') + }) + + it('reuses the cached spreadsheet context instead of re-reading it per tab', async () => { + const fetchMock = stubFetch({ + drive: { status: 200, body: { trashed: false } }, + values: { values: [['Region'], ['West']] }, + }) + const syncContext: Record = {} + + await googleSheetsConnector.listDocuments(ACCESS_TOKEN, SOURCE_CONFIG, undefined, syncContext) + await googleSheetsConnector.getDocument( + ACCESS_TOKEN, + SOURCE_CONFIG, + `${SPREADSHEET_ID}__sheet__0`, + syncContext + ) + await googleSheetsConnector.getDocument( + ACCESS_TOKEN, + SOURCE_CONFIG, + `${SPREADSHEET_ID}__sheet__7`, + syncContext + ) + + const urls = fetchMock.mock.calls.map(([input]) => String(input)) + expect( + urls.filter((url) => url.startsWith('https://www.googleapis.com/drive/')) + ).toHaveLength(1) + expect(urls.filter((url) => url.includes('/values/'))).toHaveLength(2) + }) + }) + describe('validateConfig', () => { it('rejects a spreadsheet that is already in the Drive trash', async () => { stubFetch({ drive: { status: 200, body: { trashed: true } } }) diff --git a/apps/sim/connectors/google-sheets/google-sheets.ts b/apps/sim/connectors/google-sheets/google-sheets.ts index 746bdb2bfa9..79cd8c6ad77 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.ts @@ -4,19 +4,31 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { + CONNECTOR_MAX_FILE_BYTES, + markSkipped, + parseTagDate, + readBodyWithLimit, + sizeLimitSkipReason, +} from '@/connectors/utils' const logger = createLogger('GoogleSheetsConnector') const SHEETS_API_BASE = 'https://sheets.googleapis.com/v4/spreadsheets' const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3/files' const MAX_ROWS = 10000 -const CONCURRENCY = 3 +const MAX_CONTENT_BYTES = CONNECTOR_MAX_FILE_BYTES interface SheetProperties { sheetId: number title: string index: number + /** + * `GRID` (the documented default), `OBJECT` for a chart/image tab that "has no + * grid", or `DATA_SOURCE` for a connected-data preview. Only grid-backed tabs + * hold cell values. + */ + sheetType?: 'SHEET_TYPE_UNSPECIFIED' | 'GRID' | 'OBJECT' | 'DATA_SOURCE' gridProperties?: { rowCount?: number columnCount?: number @@ -27,42 +39,88 @@ interface SpreadsheetMetadata { spreadsheetId: string properties: { title: string - locale?: string } - sheets: { properties: SheetProperties }[] + sheets?: { properties: SheetProperties }[] } /** - * Formats sheet data into an LLM-friendly text representation. - * Each row is labeled with its index and columns are identified by header names. + * True when the tab holds cell values. `OBJECT` sheets (charts, images) have no + * grid at all, so a `values.get` against one only wastes a request from the + * 60-reads-per-minute-per-user Sheets quota and can never yield content. + * An absent `sheetType` means `GRID` per the SheetProperties default. */ -function formatSheetContent(headers: string[], rows: string[][]): string { - if (headers.length === 0) return '' +function isGridSheet(sheet: SheetProperties): boolean { + return sheet.sheetType !== 'OBJECT' +} + +/** + * A1 range covering every column of the first {@link MAX_ROWS} rows of a tab. + * + * A row-only range is used deliberately instead of a column-bounded one — the A1 + * guide documents `Sheet1!1:2` as "all the cells in the first two rows of Sheet1" + * — because a sheet may hold up to 18,278 columns (column `ZZZ`, per the Drive + * size-limits support article) and a hard-coded `A1:ZZ` ceiling (column 702) + * would silently drop every column past it. Sheet names are wrapped in single + * quotes: "Single quotes are required for sheet names with spaces or special + * characters." The `''` escape for an embedded apostrophe is carried over + * unchanged from the previous range builder; it matches the Sheets UI but is not + * documented, and the guide's own `'Jon's_Data'` example leaves it unescaped. + */ +function sheetRowRange(sheetTitle: string): string { + return `'${sheetTitle.replace(/'/g, "''")}'!1:${MAX_ROWS}` +} + +/** + * Formats sheet rows into an LLM-friendly text representation, stopping as soon + * as the byte budget is exceeded so a very large tab can never be materialized + * in full. Reports `exceeded` instead of silently truncating, letting the caller + * surface the tab as a visible skipped document. + */ +function buildSheetContent( + headers: string[], + rows: string[][], + maxBytes: number +): { content: string; exceeded: boolean } { + if (headers.length === 0) return { content: '', exceeded: false } const lines: string[] = [] + let bytes = 0 for (let i = 0; i < rows.length; i++) { const row = rows[i] - lines.push(`Row ${i + 1}:`) + const rowLines = [`Row ${i + 1}:`] for (let j = 0; j < headers.length; j++) { const value = j < row.length ? row[j] : '' - lines.push(` ${headers[j]}: ${value}`) + rowLines.push(` ${headers[j]}: ${value}`) } - lines.push('') + rowLines.push('') + + for (const line of rowLines) { + bytes += Buffer.byteLength(line, 'utf8') + 1 + } + if (bytes > maxBytes) { + return { content: '', exceeded: true } + } + lines.push(...rowLines) } - return lines.join('\n').trim() + return { content: lines.join('\n').trim(), exceeded: false } } /** * Fetches all values from a single sheet tab. + * + * The response body is read against a hard byte cap rather than being parsed + * blindly: `MAX_ROWS` rows across up to 18,278 columns can produce a JSON + * payload far larger than a document the knowledge base would accept. + * `oversized` distinguishes "too big to index" from "no data". */ async function fetchSheetValues( accessToken: string, spreadsheetId: string, sheetTitle: string -): Promise { - const range = `'${sheetTitle.replace(/'/g, "''")}'!A1:ZZ${MAX_ROWS}` +): Promise<{ values: string[][]; oversized: boolean }> { + const range = sheetRowRange(sheetTitle) const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?majorDimension=ROWS&valueRenderOption=FORMATTED_VALUE` const response = await fetchWithRetry(url, { @@ -77,8 +135,23 @@ async function fetchSheetValues( throw new Error(`Failed to fetch sheet values for "${sheetTitle}": ${response.status}`) } - const data = await response.json() - return (data.values || []) as string[][] + const body = await readBodyWithLimit(response, MAX_CONTENT_BYTES) + if (!body) { + logger.warn('Sheet values response exceeded the size limit', { sheetTitle }) + return { values: [], oversized: true } + } + + const data = JSON.parse(body.toString('utf8')) + const values = (data.values || []) as string[][] + + if (values.length >= MAX_ROWS) { + logger.warn('Sheet content truncated at the row limit', { + sheetTitle, + maxRows: MAX_ROWS, + }) + } + + return { values, oversized: false } } /** @@ -88,7 +161,7 @@ async function fetchSpreadsheetMetadata( accessToken: string, spreadsheetId: string ): Promise { - const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title,properties.locale,sheets.properties` + const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title,sheets.properties` const response = await fetchWithRetry(url, { method: 'GET', @@ -121,12 +194,12 @@ export interface DriveFileMetadata { * Reports whether the spreadsheet's Drive file is in the trash. * * Trashing a Drive file does not make it unreadable: Drive keeps trashed files - * accessible by ID for 30 days before permanent deletion ("other users can - * still access the file in the owner's trash until it's permanently deleted"), - * so `spreadsheets.get` keeps succeeding and every tab keeps appearing in the + * accessible by ID before permanent deletion ("other users can still access the + * file in the owner's trash until it's permanently deleted"), so + * `spreadsheets.get` keeps succeeding and every tab keeps appearing in the * listing. Since KB deletion reconciliation only purges stored documents that * are absent from a full listing, a trashed spreadsheet's tabs would otherwise - * live in the knowledge base forever — and once the 30 days elapse the Sheets + * live in the knowledge base forever — and once the file is purged the Sheets * call 404s, the listing throws, and reconciliation never runs at all. * * Fails open: only an explicit `trashed === true` counts. A missing field or a @@ -186,8 +259,72 @@ async function fetchDriveFileMetadata( } } +interface SpreadsheetContext { + metadata: SpreadsheetMetadata + driveMetadata: DriveFileMetadata +} + +/** + * Loads the spreadsheet's Sheets metadata and Drive file metadata once per sync + * run, caching both in `syncContext`. + * + * Without the cache every deferred hydration re-reads them, so a workbook with N + * tabs costs `3N` requests instead of `N + 2` — and the Sheets API allows only 60 + * read requests per minute per user, so a few dozen tabs is enough to spend the + * whole sync in 429 backoff. + */ +async function loadSpreadsheetContext( + accessToken: string, + spreadsheetId: string, + syncContext?: Record +): Promise { + const cacheKey = `gsheets:context:${spreadsheetId}` + const cached = syncContext?.[cacheKey] as SpreadsheetContext | undefined + if (cached) return cached + + const [metadata, driveMetadata] = await Promise.all([ + fetchSpreadsheetMetadata(accessToken, spreadsheetId), + fetchDriveFileMetadata(accessToken, spreadsheetId), + ]) + + const context: SpreadsheetContext = { metadata, driveMetadata } + if (syncContext) syncContext[cacheKey] = context + return context +} + +/** + * Builds the listing stub for a tab. Shared by `listDocuments` and `getDocument` + * so the `contentHash` is byte-identical on both paths and an unchanged tab is + * never re-indexed. + */ +function sheetToStub( + spreadsheetId: string, + spreadsheetTitle: string, + sheet: SheetProperties, + modifiedTime: string | undefined, + metadata: Record +): ExternalDocument { + return { + externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`, + title: `${spreadsheetTitle} - ${sheet.title}`, + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`, + contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`, + metadata: { + spreadsheetId, + spreadsheetTitle, + sheetTitle: sheet.title, + sheetId: sheet.sheetId, + ...(modifiedTime ? { modifiedTime } : {}), + ...metadata, + }, + } +} + /** - * Converts a single sheet tab into an ExternalDocument. + * Converts a single sheet tab into an ExternalDocument with its content loaded. */ async function sheetToDocument( accessToken: string, @@ -196,17 +333,32 @@ async function sheetToDocument( sheet: SheetProperties, modifiedTime?: string ): Promise { + const stub = sheetToStub(spreadsheetId, spreadsheetTitle, sheet, modifiedTime, {}) + try { - const values = await fetchSheetValues(accessToken, spreadsheetId, sheet.title) + const { values, oversized } = await fetchSheetValues(accessToken, spreadsheetId, sheet.title) + + if (oversized) { + return markSkipped(stub, sizeLimitSkipReason(MAX_CONTENT_BYTES)) + } if (values.length === 0) { logger.info(`Skipping empty sheet: ${sheet.title}`) return null } - const headers = values[0].map((h, idx) => - typeof h === 'string' && h.trim() ? h.trim() : `Column ${idx + 1}` - ) + /** + * "Empty trailing rows and columns are omitted" from a values response, so + * rows come back ragged and a row can be wider than the header row (a blank + * header cell above populated data). Sizing the table by the widest row — + * not by the header row — keeps those columns from being dropped. + */ + const width = values.reduce((max, row) => Math.max(max, row.length), 0) + const headerRow = values[0] + const headers = Array.from({ length: width }, (_, idx) => { + const header = headerRow[idx] + return typeof header === 'string' && header.trim() ? header.trim() : `Column ${idx + 1}` + }) const dataRows = values.slice(1) if (dataRows.length === 0) { @@ -214,35 +366,36 @@ async function sheetToDocument( return null } - const content = formatSheetContent(headers, dataRows) + const { content, exceeded } = buildSheetContent(headers, dataRows, MAX_CONTENT_BYTES) + if (exceeded) { + logger.warn('Sheet content exceeded the size limit', { sheetTitle: sheet.title }) + return markSkipped(stub, sizeLimitSkipReason(MAX_CONTENT_BYTES)) + } if (!content.trim()) { return null } - const rowCount = dataRows.length - return { - externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`, - title: `${spreadsheetTitle} - ${sheet.title}`, + ...stub, content, - mimeType: 'text/plain', - sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`, - contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`, + contentDeferred: false, metadata: { - spreadsheetId, - spreadsheetTitle, - sheetTitle: sheet.title, - sheetId: sheet.sheetId, - rowCount, + ...stub.metadata, + rowCount: dataRows.length, columnCount: headers.length, - ...(modifiedTime ? { modifiedTime } : {}), }, } } catch (error) { + /** + * Empty, header-only, and oversized tabs are already resolved above. A values-fetch + * failure is transient, so it propagates and fails hydration — which is exactly what + * `listDocuments` documents as the outcome. Returning null here would instead drop + * the tab from the run with no failed row and no error recorded. + */ logger.warn(`Failed to extract content from sheet: ${sheet.title}`, { error: toError(error).message, }) - return null + throw toError(error) } } @@ -253,7 +406,7 @@ export const googleSheetsConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { const spreadsheetId = (sourceConfig.spreadsheetId as string)?.trim() if (!spreadsheetId) { @@ -262,10 +415,11 @@ export const googleSheetsConnector: ConnectorConfig = { logger.info('Fetching spreadsheet metadata', { spreadsheetId }) - const [metadata, driveMetadata] = await Promise.all([ - fetchSpreadsheetMetadata(accessToken, spreadsheetId), - fetchDriveFileMetadata(accessToken, spreadsheetId), - ]) + const { metadata, driveMetadata } = await loadSpreadsheetContext( + accessToken, + spreadsheetId, + syncContext + ) /** * A trashed spreadsheet is no longer current content, so it drops out of the @@ -283,7 +437,16 @@ export const googleSheetsConnector: ConnectorConfig = { const modifiedTime = driveMetadata.modifiedTime const sheetFilter = (sourceConfig.sheetFilter as string) || 'all' - let sheets = metadata.sheets.map((s) => s.properties) + /** + * Ordered by `index` (the tab's position in the UI) so "first sheet only" + * resolves to the leftmost tab regardless of response ordering. Sorted on a + * copy — the array belongs to the cached spreadsheet context. + */ + let sheets = (metadata.sheets ?? []) + .map((s) => s.properties) + .filter(isGridSheet) + .sort((a, b) => (a.index ?? 0) - (b.index ?? 0)) + if (sheetFilter === 'first' && sheets.length > 0) { sheets = [sheets[0]] } @@ -293,25 +456,20 @@ export const googleSheetsConnector: ConnectorConfig = { sheetCount: sheets.length, }) - const documents: ExternalDocument[] = sheets.map((sheet) => ({ - externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`, - title: `${metadata.properties.title} - ${sheet.title}`, - content: '', - contentDeferred: true, - mimeType: 'text/plain', - sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`, - contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`, - metadata: { - spreadsheetId, - spreadsheetTitle: metadata.properties.title, - sheetTitle: sheet.title, - sheetId: sheet.sheetId, + const documents = sheets.map((sheet) => + sheetToStub(spreadsheetId, metadata.properties.title, sheet, modifiedTime, { rowCount: sheet.gridProperties?.rowCount, columnCount: sheet.gridProperties?.columnCount, - ...(modifiedTime ? { modifiedTime } : {}), - }, - })) + }) + ) + /** + * No `listingCapped`: every non-object tab of the configured spreadsheet is + * listed on every run, so an absent document genuinely no longer exists and + * must reconcile. `sheetFilter: 'first'` is an intentional scope filter, and + * a per-tab values failure only fails hydration — the tab still appears in + * the listing, so it is never a candidate for deletion. + */ return { documents, hasMore: false, @@ -321,10 +479,11 @@ export const googleSheetsConnector: ConnectorConfig = { getDocument: async ( accessToken: string, sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { const parts = externalId.split('__sheet__') - if (parts.length !== 2) { + if (parts.length !== 2 || !/^\d+$/.test(parts[1])) { logger.warn('Invalid external ID format', { externalId }) return null } @@ -332,18 +491,9 @@ export const googleSheetsConnector: ConnectorConfig = { const spreadsheetId = parts[0] const sheetId = Number(parts[1]) - if (Number.isNaN(sheetId)) { - logger.warn('Invalid sheet ID in external ID', { externalId }) - return null - } - - let metadata: SpreadsheetMetadata - let driveMetadata: DriveFileMetadata + let context: SpreadsheetContext try { - ;[metadata, driveMetadata] = await Promise.all([ - fetchSpreadsheetMetadata(accessToken, spreadsheetId), - fetchDriveFileMetadata(accessToken, spreadsheetId), - ]) + context = await loadSpreadsheetContext(accessToken, spreadsheetId, syncContext) } catch (error) { const message = toError(error).message if (message.includes('404')) { @@ -354,27 +504,27 @@ export const googleSheetsConnector: ConnectorConfig = { } /** Mirrors the listing: a trashed spreadsheet is still readable but no longer current. */ - if (isTrashedDriveFile(driveMetadata)) { + if (isTrashedDriveFile(context.driveMetadata)) { logger.info('Spreadsheet is in the Drive trash', { spreadsheetId }) return null } - const sheetEntry = metadata.sheets.find((s) => s.properties.sheetId === sheetId) + const sheetEntry = (context.metadata.sheets ?? []).find( + (s) => s.properties.sheetId === sheetId && isGridSheet(s.properties) + ) if (!sheetEntry) { logger.info('Sheet not found in spreadsheet', { spreadsheetId, sheetId }) return null } - const doc = await sheetToDocument( + return sheetToDocument( accessToken, spreadsheetId, - metadata.properties.title, + context.metadata.properties.title, sheetEntry.properties, - driveMetadata.modifiedTime + context.driveMetadata.modifiedTime ) - if (!doc) return null - return { ...doc, contentDeferred: false } }, validateConfig: async ( diff --git a/apps/sim/connectors/google-slides/google-slides.ts b/apps/sim/connectors/google-slides/google-slides.ts index 3f776360cd5..cd957564e99 100644 --- a/apps/sim/connectors/google-slides/google-slides.ts +++ b/apps/sim/connectors/google-slides/google-slides.ts @@ -5,15 +5,31 @@ import { googleSlidesConnectorMeta } from '@/connectors/google-slides/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { buildDriveParentsClause, + CONNECTOR_MAX_FILE_BYTES, + ConnectorFileTooLargeError, joinTagArray, + markSkipped, parseMultiValue, parseTagDate, + readBodyWithLimit, + sizeLimitSkipReason, } from '@/connectors/utils' const logger = createLogger('GoogleSlidesConnector') const PRESENTATION_MIME_TYPE = 'application/vnd.google-apps.presentation' +/** Drive `files.list` page size. The API caps `pageSize` at 1000. */ +const PAGE_SIZE = 100 + +/** + * Ceiling for the raw `presentations.get` JSON body. The structured response is + * far larger than the plain text it yields (every run carries its styling), so + * the cap is applied to the wire body as a memory guard and again to the + * extracted text against `CONNECTOR_MAX_FILE_BYTES`. + */ +const MAX_SLIDES_RESPONSE_BYTES = 8 * CONNECTOR_MAX_FILE_BYTES + /** Reason recorded for a presentation whose slides contain no extractable text. */ const NO_TEXT = 'No extractable text' @@ -202,8 +218,17 @@ async function fetchPresentationContent( ) } - const presentation = (await response.json()) as SlidesPresentation - return extractTextFromPresentation(presentation, includeSpeakerNotes) + const buffer = await readBodyWithLimit(response, MAX_SLIDES_RESPONSE_BYTES) + if (!buffer) throw new ConnectorFileTooLargeError(CONNECTOR_MAX_FILE_BYTES) + + const presentation = JSON.parse(buffer.toString('utf8')) as SlidesPresentation + const text = extractTextFromPresentation(presentation, includeSpeakerNotes) + + if (Buffer.byteLength(text, 'utf8') > CONNECTOR_MAX_FILE_BYTES) { + throw new ConnectorFileTooLargeError(CONNECTOR_MAX_FILE_BYTES) + } + + return text } /** @@ -269,11 +294,23 @@ export const googleSlidesConnector: ConnectorConfig = { lastSyncAt?: Date ): Promise => { const query = buildQuery(sourceConfig, lastSyncAt) - const pageSize = 100 + const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0 + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + /** Last-page precision: never ask Drive for more files than the cap still allows. */ + const remaining = maxDocs > 0 ? Math.max(0, maxDocs - previouslyFetched) : 0 + const pageSize = remaining > 0 ? Math.min(PAGE_SIZE, remaining) : PAGE_SIZE + + /** + * `incompleteSearch` must be named in the partial-response mask — Drive's + * `fields` parameter filters the top-level response too, so omitting it + * leaves `data.incompleteSearch` permanently `undefined` and the + * reconciliation guard below dead. + */ const queryParams = new URLSearchParams({ q: query, pageSize: String(pageSize), + orderBy: 'modifiedTime desc', fields: 'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)', supportsAllDrives: 'true', @@ -316,18 +353,12 @@ export const googleSlidesConnector: ConnectorConfig = { */ const incompleteSearch = data.incompleteSearch === true - const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0 - const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 - const includeSpeakerNotes = shouldIncludeSpeakerNotes(sourceConfig) let documents = files.map((file) => fileToStub(file, includeSpeakerNotes)) let slicedSome = false - if (maxDocs > 0) { - const remaining = maxDocs - previouslyFetched - if (documents.length > remaining) { - slicedSome = true - documents = documents.slice(0, remaining) - } + if (maxDocs > 0 && documents.length > remaining) { + slicedSome = true + documents = documents.slice(0, remaining) } const totalFetched = previouslyFetched + documents.length @@ -383,7 +414,25 @@ export const googleSlidesConnector: ConnectorConfig = { if (file.mimeType !== PRESENTATION_MIME_TYPE) return null const includeSpeakerNotes = shouldIncludeSpeakerNotes(sourceConfig) - const content = await fetchPresentationContent(accessToken, file.id, includeSpeakerNotes) + + let content: string + try { + content = await fetchPresentationContent(accessToken, file.id, includeSpeakerNotes) + } catch (error) { + /** + * An oversized deck is a permanent, explainable outcome — surface it as a + * visible skipped row. Any other failure is transient and must propagate + * so the engine records a failed hydration instead of persisting an empty + * document. + */ + if (error instanceof ConnectorFileTooLargeError) { + return markSkipped( + fileToStub(file, includeSpeakerNotes), + sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) + ) + } + throw error + } /** * An image-only deck carries no extractable text. Surfacing it as a skipped @@ -392,12 +441,7 @@ export const googleSlidesConnector: ConnectorConfig = { * presentation would simply be missing and re-fetched on every sync. */ if (!content.trim()) { - return { - ...fileToStub(file, includeSpeakerNotes), - content: '', - contentDeferred: false, - skippedReason: NO_TEXT, - } + return markSkipped(fileToStub(file, includeSpeakerNotes), NO_TEXT) } return { ...fileToStub(file, includeSpeakerNotes), content, contentDeferred: false } diff --git a/apps/sim/connectors/google-vault/google-vault.ts b/apps/sim/connectors/google-vault/google-vault.ts index 7c94e6205fd..85e912ebb86 100644 --- a/apps/sim/connectors/google-vault/google-vault.ts +++ b/apps/sim/connectors/google-vault/google-vault.ts @@ -1,6 +1,10 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { + fetchWithRetry, + type RetryOptions, + VALIDATE_RETRY_OPTIONS, +} from '@/lib/knowledge/documents/utils' import { googleVaultConnectorMeta } from '@/connectors/google-vault/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { computeContentHash, parseTagDate, takeIndexableWithinCap } from '@/connectors/utils' @@ -9,7 +13,11 @@ const logger = createLogger('GoogleVaultConnector') const VAULT_API_BASE = 'https://vault.googleapis.com/v1' -/** Vault caps both `matters.list` and `matters.holds.list` page sizes at 100. */ +/** + * Vault documents a maximum of 100 for `matters.list` and `matters.holds.list`. + * `matters.savedQueries.list` documents no bound at all, so it reuses the same + * value rather than assuming a larger one is honored. + */ const PAGE_SIZE = 100 /** @@ -176,14 +184,23 @@ function enabledChildKinds(sourceConfig: Record): VaultChildKin return kinds } -async function vaultGet(accessToken: string, url: string, label: string): Promise { - return fetchWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', +async function vaultGet( + accessToken: string, + url: string, + label: string, + retryOptions?: RetryOptions +): Promise { + return fetchWithRetry( + url, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, }, - }).catch((error) => { + retryOptions + ).catch((error) => { throw new Error(`Failed to reach Google Vault (${label}): ${toError(error).message}`) }) } @@ -636,10 +653,15 @@ export const googleVaultConnector: ConnectorConfig = { logger.warn('Unrecognized Google Vault external ID', { externalId }) return null } catch (error) { + /** + * Only the explicit 404/403 checks above (and an unrecognized externalId kind) mean + * the object is genuinely gone or unreachable. Everything else is rethrown so the + * sync engine records a failed row instead of dropping the document silently. + */ logger.warn(`Failed to fetch Google Vault document ${externalId}`, { error: toError(error).message, }) - return null + throw toError(error) } }, @@ -664,15 +686,10 @@ export const googleVaultConnector: ConnectorConfig = { ? `${VAULT_API_BASE}/matters/${encodeURIComponent(matterId)}?view=BASIC` : `${VAULT_API_BASE}/matters?pageSize=1&view=BASIC` - const response = await fetchWithRetry( + const response = await vaultGet( + accessToken, url, - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }, + matterId ? 'matters.get' : 'matters.list', VALIDATE_RETRY_OPTIONS ) diff --git a/apps/sim/connectors/grain/grain.ts b/apps/sim/connectors/grain/grain.ts index dfe2c2a2a8a..564a334a3b5 100644 --- a/apps/sim/connectors/grain/grain.ts +++ b/apps/sim/connectors/grain/grain.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { grainConnectorMeta } from '@/connectors/grain/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -114,7 +114,16 @@ function isParticipantScope(value: unknown): value is ParticipantScope { * produces no `filter` (full sync). Returns undefined when no scoping is configured. * * Supported keys (verified against the in-repo Grain list_recordings tool / Public API): - * - `after_datetime` — derived from `lookbackDays`; recordings on/after the window start + * - `after_datetime` — derived from `lookbackDays`; recordings on/after the window start. + * Grain's Recording Filter table documents this field as "Only return recordings + * which `start_datetime` is *before* the selected date (inclusive)" and + * `before_datetime` as "…is *after* the selected date (exclusive)". The two + * descriptions are consistent with each other but inverted relative to the field + * names, so the pair reads as transposed upstream; nothing else on the page + * disambiguates and no example uses either key. The name-implied direction is used + * here and the ambiguity is deliberately left unresolved rather than guessed. If a + * live Grain source ever returns the complement of the requested lookback window, + * swap to `before_datetime` rather than re-deriving the timestamp. * - `participant_scope` — `internal` or `external` * - `title_search` — substring match against recording titles * - `team` — recordings belonging to the given team UUID @@ -282,7 +291,7 @@ function formatTranscriptContent( * Returns null on 404 (recording deleted/inaccessible). */ async function fetchRecording(accessToken: string, id: string): Promise { - const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}`, { + const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${encodeURIComponent(id)}`, { method: 'POST', headers: grainHeaders(accessToken), body: JSON.stringify({ include: RECORDING_INCLUDE }), @@ -304,10 +313,13 @@ async function fetchTranscript( accessToken: string, id: string ): Promise { - const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}/transcript`, { - method: 'GET', - headers: grainHeaders(accessToken), - }) + const response = await fetchWithRetry( + `${GRAIN_API_BASE}/recordings/${encodeURIComponent(id)}/transcript`, + { + method: 'GET', + headers: grainHeaders(accessToken), + } + ) if (!response.ok) { if (response.status === 404) return null @@ -379,7 +391,16 @@ export const grainConnector: ConnectorConfig = { const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxRecordings > 0 && totalFetched >= maxRecordings - if (hitLimit && syncContext) syncContext.listingCapped = true + + /** + * The cap only truncates the listing when recordings were actually withheld — + * either sliced off this page or still reachable behind a cursor. Reaching the + * cap exactly as the source is exhausted yields a complete listing, so flagging + * it would permanently block deletion reconciliation for that connector. + */ + const truncatedByCap = + hitLimit && (documents.length < allDocuments.length || Boolean(nextCursor)) + if (truncatedByCap && syncContext) syncContext.listingCapped = true const hasMore = !hitLimit && Boolean(nextCursor) @@ -390,55 +411,64 @@ export const grainConnector: ConnectorConfig = { } }, + /** + * Hydrates a listing stub with its transcript. + * + * Returns `null` only when the recording genuinely has nothing to index — a 404 + * from either fetch (recording or transcript deleted/inaccessible) or a transcript + * that has not finished processing. Transport, rate-limit, and server errors + * propagate so the sync engine records them as failed documents instead of + * reporting a clean sync that silently dropped recordings. + */ getDocument: async ( accessToken: string, _sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null - - const [recording, segments] = await Promise.all([ - fetchRecording(accessToken, externalId), - fetchTranscript(accessToken, externalId), - ]) - if (!recording) return null - if (!segments) return null - - const hasTranscript = segments.some((segment) => segment.text?.trim()) - if (!hasTranscript) { - logger.info('Transcript not yet available for Grain recording', { externalId }) - return null - } + if (!externalId) return null - const content = formatTranscriptContent(recording, segments) - - return { - externalId, - title: recordingTitle(recording), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: recording.url || undefined, - contentHash: buildContentHash(recording), - metadata: buildMetadata(recording), - } - } catch (error) { - logger.warn('Failed to get Grain recording', { - externalId, - error: toError(error).message, - }) + const [recording, segments] = await Promise.all([ + fetchRecording(accessToken, externalId), + fetchTranscript(accessToken, externalId), + ]) + if (!recording || !segments) return null + + const hasTranscript = segments.some((segment) => segment.text?.trim()) + if (!hasTranscript) { + logger.info('Transcript not yet available for Grain recording', { externalId }) return null } + + return { + externalId, + title: recordingTitle(recording), + content: formatTranscriptContent(recording, segments), + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: recording.url || undefined, + contentHash: buildContentHash(recording), + metadata: buildMetadata(recording), + } }, validateConfig: async ( accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const maxRecordings = sourceConfig.maxRecordings as string | undefined - if (maxRecordings && (Number.isNaN(Number(maxRecordings)) || Number(maxRecordings) < 0)) { - return { valid: false, error: 'Max recordings must be a non-negative number' } + const maxRecordings = sourceConfig.maxRecordings + if (maxRecordings != null && maxRecordings !== '') { + const parsed = Number(maxRecordings) + if (!Number.isFinite(parsed) || parsed < 0) { + return { valid: false, error: 'Max recordings must be a non-negative number' } + } + } + + const lookbackDays = sourceConfig.lookbackDays + if (lookbackDays != null && lookbackDays !== '') { + const parsed = Number(lookbackDays) + if (!Number.isFinite(parsed) || parsed < 0) { + return { valid: false, error: 'Lookback window must be a non-negative number of days' } + } } try { diff --git a/apps/sim/connectors/granola/granola.ts b/apps/sim/connectors/granola/granola.ts index 58023194580..def1e208f27 100644 --- a/apps/sim/connectors/granola/granola.ts +++ b/apps/sim/connectors/granola/granola.ts @@ -3,7 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { granolaConnectorMeta } from '@/connectors/granola/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' +import { htmlToPlainText, joinTagArray, looksLikeHtml, parseTagDate } from '@/connectors/utils' const logger = createLogger('GranolaConnector') @@ -85,7 +85,7 @@ interface GranolaListNotesResponse { function granolaHeaders(accessToken: string): Record { return { Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', + Accept: 'application/json', } } @@ -138,15 +138,6 @@ function parseDateFilter(sourceConfig: Record, key: string): st return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString() } -/** - * Detects whether a string contains HTML markup. Granola returns markdown for - * `summary_markdown`, but this guard lets us defensively strip tags if the API - * ever emits HTML, without mangling legitimate markdown. - */ -function looksLikeHtml(value: string): boolean { - return /<\/?[a-z][\s\S]*?>/i.test(value) -} - /** * Assembles the document content from a note's title and summary. Prefers the * markdown summary, falling back to plain-text summary. HTML is stripped only @@ -271,20 +262,34 @@ export const granolaConnector: ConnectorConfig = { const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let documents = allStubs + let capDroppedNotes = false if (maxNotes > 0) { const remaining = Math.max(0, maxNotes - prevFetched) if (allStubs.length > remaining) { documents = allStubs.slice(0, remaining) + capDroppedNotes = true } } const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched + const sourceHasMore = Boolean(data.hasMore) && Boolean(nextCursor) const hitLimit = maxNotes > 0 && totalFetched >= maxNotes - if (hitLimit && syncContext) syncContext.listingCapped = true - const hasMore = !hitLimit && Boolean(data.hasMore) && Boolean(nextCursor) + /** + * Only report the listing as capped when the cap actually hid notes that + * still exist — either this page was sliced, or Granola reports further + * pages. A cap that lands exactly on the last note leaves the listing + * complete, and flagging it there would block deletion reconciliation on + * every ordinary sync, stranding notes deleted in Granola in the knowledge + * base indefinitely. + */ + if (syncContext && hitLimit && (capDroppedNotes || sourceHasMore)) { + syncContext.listingCapped = true + } + + const hasMore = !hitLimit && sourceHasMore return { documents, @@ -309,7 +314,7 @@ export const granolaConnector: ConnectorConfig = { }) if (!response.ok) { - if (response.status === 404 || response.status === 410) return null + if (response.status === 404) return null throw new Error(`Failed to fetch Granola note: ${response.status}`) } @@ -349,11 +354,19 @@ export const granolaConnector: ConnectorConfig = { }, } } catch (error) { + /** + * Only a confirmed 404 above returns null (the note is gone, or was never + * summarized — Granola 404s both). Everything else — 429 rate limiting, + * 5xx, network faults — is rethrown so the sync engine records a failed + * row and preserves the already-indexed document, instead of silently + * dropping a note that still exists. Granola's documented 5 req/s + * sustained limit makes transient 429s a realistic outcome on large syncs. + */ logger.warn('Failed to get Granola note', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/greenhouse/greenhouse.test.ts b/apps/sim/connectors/greenhouse/greenhouse.test.ts new file mode 100644 index 00000000000..aec03f5dab0 --- /dev/null +++ b/apps/sim/connectors/greenhouse/greenhouse.test.ts @@ -0,0 +1,342 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { greenhouseConnector } from '@/connectors/greenhouse/greenhouse' + +const ACCESS_TOKEN = 'test-key' + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', ...headers }, + }) +} + +/** A `Link` header advertising another page, exactly as Harvest emits it. */ +const NEXT_PAGE_LINK = { + link: '; rel="next"', +} + +function candidateFixture(id: number, overrides: Record = {}) { + return { + id, + first_name: 'Ada', + last_name: `Lovelace ${id}`, + company: 'Analytical Engines', + title: 'Engineer', + created_at: '2024-01-01T00:00:00.000Z', + updated_at: '2024-02-01T00:00:00.000Z', + last_activity: '2024-03-01T00:00:00.000Z', + email_addresses: [{ value: `ada${id}@example.com`, type: 'personal' }], + tags: ['referral'], + application_ids: [900 + id], + applications: [{ id: 900 + id, applied_at: '2024-01-02T00:00:00.000Z' }], + ...overrides, + } +} + +function requestUrl(callIndex = 0): URL { + const call = mockFetch.mock.calls[callIndex] + if (!call) throw new Error(`No fetch call at index ${callIndex}`) + return new URL(String(call[0])) +} + +beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('greenhouseConnector auth', () => { + it('sends Basic auth as base64 of the api key with an empty password', async () => { + mockFetch.mockResolvedValue(jsonResponse([])) + + await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + const init = mockFetch.mock.calls[0][1] as RequestInit + const header = (init.headers as Record).Authorization + expect(header).toBe(`Basic ${Buffer.from('test-key:').toString('base64')}`) + expect(Buffer.from(header.replace('Basic ', ''), 'base64').toString()).toBe('test-key:') + }) +}) + +describe('greenhouseConnector.listDocuments pagination', () => { + it('keeps per_page constant across pages so page-number paging cannot slide', async () => { + const fullPage = Array.from({ length: 500 }, (_, i) => candidateFixture(i + 1)) + mockFetch.mockImplementation(() => Promise.resolve(jsonResponse(fullPage, 200, NEXT_PAGE_LINK))) + const syncContext: Record = {} + + const first = await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '600' }, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '600' }, + first.nextCursor, + syncContext + ) + + expect(requestUrl(0).searchParams.get('per_page')).toBe('500') + expect(requestUrl(1).searchParams.get('per_page')).toBe('500') + expect(requestUrl(1).searchParams.get('page')).toBe('2') + }) + + it('forwards the configured scope filters', async () => { + mockFetch.mockResolvedValue(jsonResponse([])) + + await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { + jobId: '123456', + createdAfter: '2024-01-01T00:00:00Z', + createdBefore: '2024-12-31T23:59:59Z', + }, + undefined, + {} + ) + + const params = requestUrl().searchParams + expect(params.get('job_id')).toBe('123456') + expect(params.get('created_after')).toBe('2024-01-01T00:00:00Z') + expect(params.get('created_before')).toBe('2024-12-31T23:59:59Z') + }) +}) + +describe('greenhouseConnector listingCapped', () => { + it('leaves listingCapped unset when the source is exhausted', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1), candidateFixture(2)])) + const syncContext: Record = {} + + const result = await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '10' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('leaves listingCapped unset when the cap lands exactly on exhaustion', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1), candidateFixture(2)])) + const syncContext: Record = {} + + await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when the cap hides candidates on the same page', async () => { + mockFetch.mockResolvedValue( + jsonResponse([candidateFixture(1), candidateFixture(2), candidateFixture(3)]) + ) + const syncContext: Record = {} + + const result = await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when the cap stops paging while a next page exists', async () => { + mockFetch.mockResolvedValue( + jsonResponse([candidateFixture(1), candidateFixture(2)], 200, NEXT_PAGE_LINK) + ) + const syncContext: Record = {} + + const result = await greenhouseConnector.listDocuments( + ACCESS_TOKEN, + { maxCandidates: '2' }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('does not flag listingCapped when no cap is configured', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const syncContext: Record = {} + + await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, syncContext) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('throws instead of reporting an empty listing when the API fails', async () => { + mockFetch.mockResolvedValue(jsonResponse({ error: 'nope' }, 500)) + + await expect( + greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + ).rejects.toThrow('500') + }) +}) + +describe('greenhouseConnector.getDocument', () => { + /** Mocks the candidate + activity feed + scorecards calls a hydration makes. */ + function mockHydration(scorecardsResponse: Response) { + mockFetch.mockImplementation((url: string) => { + const href = String(url) + if (href.includes('/activity_feed')) { + return Promise.resolve( + jsonResponse({ notes: [{ body: '

Great chat

', created_at: '2024-02-02' }] }) + ) + } + if (href.includes('/scorecards')) return Promise.resolve(scorecardsResponse) + return Promise.resolve(jsonResponse(candidateFixture(1))) + }) + } + + it('produces the same contentHash as the listing stub', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const listed = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + const stub = listed.documents[0] + expect(stub.contentDeferred).toBe(true) + expect(stub.content).toBe('') + + mockFetch.mockReset() + mockHydration(jsonResponse([{ id: 1, interview: 'Onsite', overall_recommendation: 'yes' }])) + + const full = await greenhouseConnector.getDocument(ACCESS_TOKEN, {}, stub.externalId) + + expect(full?.contentHash).toBe(stub.contentHash) + expect(full?.contentDeferred).toBe(false) + expect(full?.content).toContain('Great chat') + expect(full?.content).not.toContain('

') + }) + + it('folds last_activity into the hash so feed-only changes are detected', async () => { + mockFetch.mockResolvedValue( + jsonResponse([candidateFixture(1, { last_activity: '2024-09-09T00:00:00.000Z' })]) + ) + const moved = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const original = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + expect(moved.documents[0].contentHash).not.toBe(original.documents[0].contentHash) + }) + + it('marks the hash partial when scorecards could not be fetched, so the next sync retries', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const listed = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + const stubHash = listed.documents[0].contentHash + + mockFetch.mockReset() + mockHydration(jsonResponse({ error: 'nope' }, 500)) + const partial = await greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1') + + mockFetch.mockReset() + mockHydration(jsonResponse([])) + const complete = await greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1') + + expect(partial?.contentHash).not.toBe(stubHash) + expect(partial?.content).toContain('Great chat') + expect(complete?.contentHash).toBe(stubHash) + }) + + it('treats a 404 scorecard list as a complete absence, not a partial fetch', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const listed = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + mockFetch.mockReset() + mockHydration(jsonResponse({}, 404)) + const doc = await greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1') + + expect(doc?.contentHash).toBe(listed.documents[0].contentHash) + }) + + it('treats a 403 scorecard list as settled, so a permission gap cannot re-hydrate forever', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const listed = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + mockFetch.mockReset() + mockHydration(jsonResponse({ error: 'no access' }, 403)) + const doc = await greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1') + + expect(doc?.contentHash).toBe(listed.documents[0].contentHash) + }) + + it('returns null for a deleted candidate', async () => { + mockFetch.mockResolvedValue(jsonResponse({}, 404)) + + await expect(greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1')).resolves.toBeNull() + }) + + it('throws when the API fails so the sync engine records a failed row', async () => { + mockFetch.mockResolvedValue(jsonResponse({ error: 'nope' }, 500)) + + await expect(greenhouseConnector.getDocument(ACCESS_TOKEN, {}, '1')).rejects.toThrow('500') + }) +}) + +describe('greenhouseConnector.validateConfig', () => { + it('rejects a non-numeric job id without calling the API', async () => { + const result = await greenhouseConnector.validateConfig(ACCESS_TOKEN, { jobId: 'Engineering' }) + + expect(result.valid).toBe(false) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('rejects an unparseable timestamp without calling the API', async () => { + const result = await greenhouseConnector.validateConfig(ACCESS_TOKEN, { + createdAfter: 'last tuesday', + }) + + expect(result.valid).toBe(false) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('accepts a valid config and probes with a single record', async () => { + mockFetch.mockResolvedValue(jsonResponse([])) + + const result = await greenhouseConnector.validateConfig(ACCESS_TOKEN, { + jobId: '123456', + createdAfter: '2024-01-01T00:00:00Z', + }) + + expect(result.valid).toBe(true) + expect(requestUrl().searchParams.get('per_page')).toBe('1') + }) +}) + +describe('greenhouseConnector.mapTags', () => { + it('only emits tag ids declared in tagDefinitions', async () => { + mockFetch.mockResolvedValue(jsonResponse([candidateFixture(1)])) + const listed = await greenhouseConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {}) + + const declared = new Set((greenhouseConnector.tagDefinitions ?? []).map((t) => t.id)) + const mapped = greenhouseConnector.mapTags?.(listed.documents[0].metadata ?? {}) + + expect(Object.keys(mapped ?? {}).length).toBeGreaterThan(0) + for (const key of Object.keys(mapped ?? {})) { + expect(declared.has(key)).toBe(true) + } + expect(mapped?.updatedAt).toBeInstanceOf(Date) + expect(mapped?.lastActivity).toBeInstanceOf(Date) + }) +}) diff --git a/apps/sim/connectors/greenhouse/greenhouse.ts b/apps/sim/connectors/greenhouse/greenhouse.ts index 1159f4c2bbe..e4b78e007fa 100644 --- a/apps/sim/connectors/greenhouse/greenhouse.ts +++ b/apps/sim/connectors/greenhouse/greenhouse.ts @@ -17,10 +17,17 @@ const GREENHOUSE_API_BASE = 'https://harvest.greenhouse.io/v1' */ const MAX_APPLICATIONS_FOR_SCORECARDS = 10 +/** Concurrent per-application scorecard requests issued during one getDocument call. */ +const SCORECARD_FETCH_CONCURRENCY = 5 + /** * Greenhouse Harvest allows up to 500 candidates per page. We page through the * full list using the `page` query parameter and stop when the `Link` response * header no longer advertises a `rel="next"` relationship. + * + * `per_page` is deliberately constant across pages: `page` selects the n-th chunk + * *of `per_page` records*, so shrinking it on a later page would slide the window + * backwards and skip candidates. */ const CANDIDATES_PER_PAGE = 500 @@ -165,13 +172,16 @@ function candidateDisplayName(candidate: GreenhouseCandidate): string { /** * Computes the metadata-based content hash for a candidate. Both the listing stub * and `getDocument` use the same formula so the sync engine can detect changes - * without downloading the deferred content. Greenhouse advances a candidate's - * `updated_at` when the candidate record changes; profile-affecting activity - * (notes, emails, stage changes, scorecard submissions) typically also touches it, - * which is why `updated_after` listing and this hash track the same field. + * without downloading the deferred content. + * + * `last_activity` is mixed in alongside `updated_at` because the document body is + * assembled from the activity feed and scorecards, and Greenhouse does not document + * that appending a note, email, or scorecard advances `updated_at`. Both fields are + * returned by the list and the single-candidate endpoints, so the stub and + * `getDocument` always agree. */ -function buildContentHash(id: number, updatedAt?: string | null): string { - return `greenhouse:${id}:${updatedAt ?? ''}` +function buildContentHash(candidate: GreenhouseCandidate): string { + return `greenhouse:${candidate.id}:${candidate.updated_at ?? ''}:${candidate.last_activity ?? ''}` } /** @@ -256,7 +266,7 @@ function candidateToStub(candidate: GreenhouseCandidate): ExternalDocument { contentDeferred: true, mimeType: 'text/plain', sourceUrl: buildSourceUrl(candidate.id), - contentHash: buildContentHash(candidate.id, candidate.updated_at), + contentHash: buildContentHash(candidate), metadata: buildMetadata(candidate), } } @@ -435,41 +445,85 @@ async function fetchActivityFeed(accessToken: string, id: string): Promise { - const all: GreenhouseScorecard[] = [] - - for (const applicationId of applicationIds.slice(0, MAX_APPLICATIONS_FOR_SCORECARDS)) { - try { - const response = await fetchWithRetry( - `${GREENHOUSE_API_BASE}/applications/${applicationId}/scorecards`, - { - method: 'GET', - headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, - } - ) + applicationId: number +): Promise<{ scorecards: GreenhouseScorecard[]; complete: boolean }> { + try { + const response = await fetchWithRetry( + `${GREENHOUSE_API_BASE}/applications/${applicationId}/scorecards`, + { + method: 'GET', + headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' }, + } + ) - if (!response.ok) { - if (response.status === 404) continue - throw new Error(`Failed to fetch Greenhouse scorecards: ${response.status}`) + if (!response.ok) { + if (response.status === 404 || response.status === 403) { + return { scorecards: [], complete: true } } + throw new Error(`Failed to fetch Greenhouse scorecards: ${response.status}`) + } - const data = (await response.json()) as GreenhouseScorecard[] - if (Array.isArray(data)) all.push(...data) - } catch (error) { - logger.warn('Failed to fetch scorecards for application', { - applicationId, - error: toError(error).message, - }) + const data = (await response.json()) as GreenhouseScorecard[] + return { scorecards: Array.isArray(data) ? data : [], complete: true } + } catch (error) { + logger.warn('Failed to fetch scorecards for application', { + applicationId, + error: toError(error).message, + }) + return { scorecards: [], complete: false } + } +} + +/** + * Fetches all scorecards across a candidate's applications, in bounded-concurrency + * batches. Application order is preserved so the rendered content is stable across + * syncs. + * + * `complete` is false only when a transient failure hid feedback that still + * exists. The deliberate {@link MAX_APPLICATIONS_FOR_SCORECARDS} bound keeps it + * true: it truncates identically on every run, so marking it partial would + * re-hydrate the same candidate forever without ever converging. + */ +async function fetchScorecards( + accessToken: string, + applicationIds: number[] +): Promise<{ scorecards: GreenhouseScorecard[]; complete: boolean }> { + const bounded = applicationIds.slice(0, MAX_APPLICATIONS_FOR_SCORECARDS) + let complete = true + if (bounded.length < applicationIds.length) { + logger.warn('Candidate exceeds the scorecard application bound; later applications skipped', { + applications: applicationIds.length, + fetched: bounded.length, + }) + } + + const scorecards: GreenhouseScorecard[] = [] + for (let i = 0; i < bounded.length; i += SCORECARD_FETCH_CONCURRENCY) { + const batch = bounded.slice(i, i + SCORECARD_FETCH_CONCURRENCY) + const results = await Promise.all( + batch.map((applicationId) => fetchScorecardsForApplication(accessToken, applicationId)) + ) + for (const result of results) { + scorecards.push(...result.scorecards) + if (!result.complete) complete = false } } - return all + return { scorecards, complete } } /** @@ -502,6 +556,9 @@ export const greenhouseConnector: ConnectorConfig = { const createdBefore = typeof sourceConfig.createdBefore === 'string' ? sourceConfig.createdBefore.trim() : '' + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = maxCandidates > 0 ? Math.max(0, maxCandidates - prevFetched) : 0 + const queryParams = new URLSearchParams({ per_page: String(CANDIDATES_PER_PAGE), page: String(page), @@ -537,13 +594,9 @@ export const greenhouseConnector: ConnectorConfig = { const candidates = Array.isArray(data) ? data : [] const linkHasNext = hasNextPage(response.headers.get('link')) - const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let pageCandidates = candidates - if (maxCandidates > 0) { - const remaining = Math.max(0, maxCandidates - prevFetched) - if (pageCandidates.length > remaining) { - pageCandidates = pageCandidates.slice(0, remaining) - } + if (maxCandidates > 0 && pageCandidates.length > remaining) { + pageCandidates = pageCandidates.slice(0, remaining) } const documents = pageCandidates.map(candidateToStub) @@ -551,10 +604,20 @@ export const greenhouseConnector: ConnectorConfig = { const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates - if (hitLimit && syncContext) syncContext.listingCapped = true - const hasMore = !hitLimit && linkHasNext + /** + * Deletion reconciliation hard-deletes every stored document missing from the + * listing, so the cap flag must be raised only when candidates that still exist + * were genuinely left unlisted: either this page was trimmed to fit the cap, or + * the cap stopped paging while Greenhouse still advertises a `rel="next"` page. + * A run that reaches the cap exactly as the source is exhausted is a complete + * listing and must stay reconcilable. + */ + if (syncContext && (pageCandidates.length < candidates.length || (hitLimit && linkHasNext))) { + syncContext.listingCapped = true + } + return { documents, nextCursor: hasMore ? String(page + 1) : undefined, @@ -577,7 +640,7 @@ export const greenhouseConnector: ConnectorConfig = { ? candidate.application_ids : [] - const [feed, scorecards] = await Promise.all([ + const [feed, { scorecards, complete }] = await Promise.all([ fetchActivityFeed(accessToken, externalId), fetchScorecards(accessToken, applicationIds), ]) @@ -585,6 +648,17 @@ export const greenhouseConnector: ConnectorConfig = { const content = formatContent(candidate, feed, scorecards) if (!content.trim()) return null + /** + * A failed scorecard fetch yields usable but incomplete content. Storing it + * under the canonical hash would freeze the gap in place until the candidate + * itself changes, so a partial marker is appended instead: it never matches + * the list stub's hash, so the next sync re-hydrates and self-heals once + * Greenhouse responds. + */ + const contentHash = complete + ? buildContentHash(candidate) + : `${buildContentHash(candidate)}:partial` + return { externalId: String(candidate.id), title: candidateDisplayName(candidate), @@ -592,15 +666,20 @@ export const greenhouseConnector: ConnectorConfig = { contentDeferred: false, mimeType: 'text/plain', sourceUrl: buildSourceUrl(candidate.id), - contentHash: buildContentHash(candidate.id, candidate.updated_at), + contentHash, metadata: buildMetadata(candidate), } } catch (error) { + /** + * `fetchCandidate` already returns null on a 404, so a thrown error here is + * transient (429, 5xx, network) and must propagate for the engine to record a + * failed row instead of dropping a candidate that still exists. + */ logger.warn('Failed to get Greenhouse candidate', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, @@ -613,6 +692,25 @@ export const greenhouseConnector: ConnectorConfig = { return { valid: false, error: 'Max candidates must be a non-negative number' } } + const jobId = sourceConfig.jobId + if (typeof jobId === 'string' && jobId.trim() && !/^\d+$/.test(jobId.trim())) { + return { valid: false, error: 'Job ID must be a numeric Greenhouse job ID (e.g. 123456)' } + } + + const timestampFields = [ + ['createdAfter', 'Created After'], + ['createdBefore', 'Created Before'], + ] as const + for (const [field, label] of timestampFields) { + const value = sourceConfig[field] + if (typeof value === 'string' && value.trim() && Number.isNaN(Date.parse(value.trim()))) { + return { + valid: false, + error: `${label} must be a valid ISO 8601 timestamp (e.g. 2024-01-01T00:00:00Z)`, + } + } + } + try { const response = await fetchWithRetry( `${GREENHOUSE_API_BASE}/candidates?per_page=1`, diff --git a/apps/sim/connectors/hubspot/hubspot.ts b/apps/sim/connectors/hubspot/hubspot.ts index 28eb0ed89a0..936ca11fd73 100644 --- a/apps/sim/connectors/hubspot/hubspot.ts +++ b/apps/sim/connectors/hubspot/hubspot.ts @@ -3,12 +3,38 @@ import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { hubspotConnectorMeta } from '@/connectors/hubspot/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { htmlToPlainText, looksLikeHtml, parseTagDate } from '@/connectors/utils' const logger = createLogger('HubSpotConnector') const BASE_URL = 'https://api.hubapi.com' -const PAGE_SIZE = 100 + +/** Max `limit` for `GET /crm/v3/objects/{objectType}` (list endpoint). */ +const LIST_PAGE_SIZE = 100 + +/** Max `limit` for `POST /crm/v3/objects/{objectType}/search`. */ +const SEARCH_PAGE_SIZE = 200 + +/** + * The CRM Search API is capped at 10,000 total results per query — HubSpot + * documents that "attempting to page beyond 10,000 will result in a 400 error". + * No equivalent ceiling is documented for the list endpoint, so search is used + * only when the configured record cap provably fits inside the search ceiling + * (which buys most-recently-modified-first ordering for capped syncs); every + * uncapped sync pages the list endpoint instead. + */ +const SEARCH_RESULT_CAP = 10_000 + +/** Fallback UI host when the account's `uiDomain` cannot be resolved. */ +const DEFAULT_UI_DOMAIN = 'app.hubspot.com' + +/** CRM object type IDs used in HubSpot record deep links (`/record/{typeId}/{id}`). */ +const OBJECT_TYPE_IDS: Record = { + contacts: '0-1', + companies: '0-2', + deals: '0-3', + tickets: '0-5', +} /** Properties to fetch per object type. */ const OBJECT_PROPERTIES: Record = { @@ -62,39 +88,68 @@ const OBJECT_PROPERTIES: Record = { ], } as const +interface PortalInfo { + portalId: string + uiDomain: string +} + /** - * Fetches the HubSpot portal ID for the authenticated account. - * Caches the result in syncContext to avoid repeated calls. + * Fetches the HubSpot account's portal ID and UI domain (EU portals are served + * from `app-eu1.hubspot.com`, not `app.hubspot.com`), caching the result — and + * any failure — in syncContext so it is fetched at most once per sync. + * + * Record deep links are cosmetic, so a failure here degrades to no `sourceUrl` + * rather than aborting the whole listing. */ -async function getPortalId( +async function getPortalInfo( accessToken: string, syncContext?: Record -): Promise { - if (syncContext?.portalId) { - return syncContext.portalId as string +): Promise { + if (syncContext?.portalInfo) { + return syncContext.portalInfo as PortalInfo + } + if (syncContext?.portalInfoUnavailable) { + return null } - const response = await fetchWithRetry(`${BASE_URL}/account-info/v3/details`, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) + try { + const response = await fetchWithRetry(`${BASE_URL}/account-info/v3/details`, { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + }) - if (!response.ok) { - const errorText = await response.text() - throw new Error(`Failed to fetch HubSpot portal ID: ${response.status} - ${errorText}`) - } + if (!response.ok) { + const errorText = await response.text() + logger.warn('Failed to fetch HubSpot account details; record links will be omitted', { + status: response.status, + error: errorText, + }) + if (syncContext) syncContext.portalInfoUnavailable = true + return null + } - const data = await response.json() - const portalId = String(data.portalId) + const data = (await response.json()) as { portalId?: number | string; uiDomain?: string } + if (data.portalId === undefined || data.portalId === null) { + if (syncContext) syncContext.portalInfoUnavailable = true + return null + } - if (syncContext) { - syncContext.portalId = portalId + const portalInfo: PortalInfo = { + portalId: String(data.portalId), + uiDomain: data.uiDomain || DEFAULT_UI_DOMAIN, + } + if (syncContext) syncContext.portalInfo = portalInfo + return portalInfo + } catch (error) { + logger.warn('Failed to fetch HubSpot account details; record links will be omitted', { + error: getErrorMessage(error), + }) + if (syncContext) syncContext.portalInfoUnavailable = true + return null } - - return portalId } /** @@ -119,6 +174,19 @@ function buildRecordTitle(objectType: string, properties: Record`), + * and a false positive here does not pass the value through — `htmlToPlainText` + * would delete the bracketed span and collapse the value's line structure. + */ +function toPlainTextValue(value: string): string { + return looksLikeHtml(value) ? htmlToPlainText(value) : value +} + /** * Builds a plain-text representation of a CRM record's properties for indexing. */ @@ -131,7 +199,7 @@ function buildRecordContent(objectType: string, properties: Record c.toUpperCase()) - parts.push(`${label}: ${value}`) + parts.push(`${label}: ${toPlainTextValue(value)}`) } } @@ -144,7 +212,7 @@ function buildRecordContent(objectType: string, properties: Record, objectType: string, - portalId: string + portal: PortalInfo | null ): ExternalDocument { const id = record.id as string const properties = (record.properties || {}) as Record @@ -155,13 +223,27 @@ function recordToDocument( const lastModified = properties.lastmodifieddate || properties.hs_lastmodifieddate || properties.createdate + const objectTypeId = OBJECT_TYPE_IDS[objectType] + const sourceUrl = + portal && objectTypeId + ? `https://${portal.uiDomain}/contacts/${portal.portalId}/record/${objectTypeId}/${id}` + : undefined + return { externalId: id, title, content, mimeType: 'text/plain', - sourceUrl: `https://app.hubspot.com/contacts/${portalId}/record/${objectType}/${id}`, - contentHash: `hubspot:${id}:${lastModified ?? ''}`, + sourceUrl, + /** + * The `v2` namespace is a one-time invalidation. The hash is metadata-only + * and this connector sets neither `contentDeferred` nor + * `rehydrateOnFullSync`, so a stored record whose `lastmodifieddate` has not + * moved can never be re-indexed — not even by a forced full resync. Without + * the bump, existing documents would keep the raw HTML that rich-text + * properties return, since `htmlToPlainText` is only applied on write. + */ + contentHash: `hubspot:v2:${id}:${lastModified ?? ''}`, metadata: { objectType, owner: properties.hubspot_owner_id || undefined, @@ -181,34 +263,71 @@ export const hubspotConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const objectType = sourceConfig.objectType as string - const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0 - const properties = OBJECT_PROPERTIES[objectType] || [] + const properties = OBJECT_PROPERTIES[objectType] + if (!properties) { + throw new Error(`Unsupported HubSpot object type: ${objectType}`) + } - const portalId = await getPortalId(accessToken, syncContext) + const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0 + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = maxRecords > 0 ? Math.max(maxRecords - previouslyFetched, 0) : 0 + + const portal = await getPortalInfo(accessToken, syncContext) + + /** + * The search endpoint sorts by last-modified (so a capped sync keeps the most + * recent records) but cannot page past 10,000 results. An uncapped sync — or a + * cap larger than that ceiling — uses the list endpoint, which pages the full + * object without a ceiling. + */ + const useSearch = maxRecords > 0 && maxRecords <= SEARCH_RESULT_CAP + const pageMax = useSearch ? SEARCH_PAGE_SIZE : LIST_PAGE_SIZE + const limit = maxRecords > 0 ? Math.min(pageMax, Math.max(1, remaining)) : pageMax + + logger.info(`Listing HubSpot ${objectType}`, { cursor, useSearch, limit }) + + let response: Response + if (useSearch) { + const sortProperty = objectType === 'contacts' ? 'lastmodifieddate' : 'hs_lastmodifieddate' + const searchBody: Record = { + properties, + sorts: [{ propertyName: sortProperty, direction: 'DESCENDING' }], + limit, + } + if (cursor) searchBody.after = cursor - const sortProperty = objectType === 'contacts' ? 'lastmodifieddate' : 'hs_lastmodifieddate' + response = await fetchWithRetry( + `${BASE_URL}/crm/v3/objects/${encodeURIComponent(objectType)}/search`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + body: JSON.stringify(searchBody), + } + ) + } else { + const params = new URLSearchParams({ + limit: String(limit), + properties: properties.join(','), + archived: 'false', + }) + if (cursor) params.set('after', cursor) - const searchBody: Record = { - properties, - sorts: [{ propertyName: sortProperty, direction: 'DESCENDING' }], - limit: PAGE_SIZE, - } - if (cursor) { - searchBody.after = cursor + response = await fetchWithRetry( + `${BASE_URL}/crm/v3/objects/${encodeURIComponent(objectType)}?${params.toString()}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + } + ) } - logger.info(`Listing HubSpot ${objectType}`, { cursor }) - - const response = await fetchWithRetry(`${BASE_URL}/crm/v3/objects/${objectType}/search`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(searchBody), - }) - if (!response.ok) { const errorText = await response.text() logger.error(`Failed to list HubSpot ${objectType}`, { @@ -223,16 +342,12 @@ export const hubspotConnector: ConnectorConfig = { const paging = data.paging as { next?: { after?: string } } | undefined const nextCursor = paging?.next?.after - const documents: ExternalDocument[] = results.map((record) => - recordToDocument(record, objectType, portalId) + let documents: ExternalDocument[] = results.map((record) => + recordToDocument(record, objectType, portal) ) - const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 - if (maxRecords > 0) { - const remaining = maxRecords - previouslyFetched - if (documents.length > remaining) { - documents.splice(remaining) - } + if (maxRecords > 0 && documents.length > remaining) { + documents = documents.slice(0, remaining) } const totalFetched = previouslyFetched + documents.length @@ -240,7 +355,27 @@ export const hubspotConnector: ConnectorConfig = { syncContext.totalDocsFetched = totalFetched } - const hasMore = Boolean(nextCursor) && (maxRecords <= 0 || totalFetched < maxRecords) + const moreAvailable = Boolean(nextCursor) + /** + * The search path can never reach its own 10,000-result ceiling: `useSearch` + * only holds when `maxRecords <= SEARCH_RESULT_CAP`, so the record cap always + * stops paging first. The cap is therefore the only stop condition here. + */ + const hasMore = moreAvailable && !(maxRecords > 0 && totalFetched >= maxRecords) + + /** + * Stopping while the source still has pages leaves the listing partial. The + * sync engine hard-deletes stored documents absent from a full listing, so + * the cap must suppress deletion reconciliation. Genuine exhaustion (no + * `paging.next.after`) leaves the flag unset so deletions still reconcile. + */ + if (moreAvailable && !hasMore && syncContext) { + syncContext.listingCapped = true + logger.warn(`HubSpot ${objectType} listing capped before source exhaustion`, { + totalFetched, + maxRecords, + }) + } return { documents, @@ -256,20 +391,19 @@ export const hubspotConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const objectType = sourceConfig.objectType as string - const properties = OBJECT_PROPERTIES[objectType] || [] - - let portalId = syncContext?.portalId as string | undefined - if (!portalId) { - portalId = await getPortalId(accessToken) - if (syncContext) syncContext.portalId = portalId + const properties = OBJECT_PROPERTIES[objectType] + if (!properties) { + throw new Error(`Unsupported HubSpot object type: ${objectType}`) } - const params = new URLSearchParams() - for (const prop of properties) { - params.append('properties', prop) - } + const portal = await getPortalInfo(accessToken, syncContext) + + const params = new URLSearchParams({ + properties: properties.join(','), + archived: 'false', + }) - const url = `${BASE_URL}/crm/v3/objects/${objectType}/${externalId}?${params.toString()}` + const url = `${BASE_URL}/crm/v3/objects/${encodeURIComponent(objectType)}/${encodeURIComponent(externalId)}?${params.toString()}` const response = await fetchWithRetry(url, { method: 'GET', @@ -285,7 +419,7 @@ export const hubspotConnector: ConnectorConfig = { } const record = await response.json() - return recordToDocument(record as Record, objectType, portalId) + return recordToDocument(record as Record, objectType, portal) }, validateConfig: async ( @@ -309,7 +443,7 @@ export const hubspotConnector: ConnectorConfig = { try { const response = await fetchWithRetry( - `${BASE_URL}/crm/v3/objects/${objectType}?limit=1`, + `${BASE_URL}/crm/v3/objects/${encodeURIComponent(objectType)}?limit=1`, { method: 'GET', headers: { diff --git a/apps/sim/connectors/hubspot/meta.ts b/apps/sim/connectors/hubspot/meta.ts index 2660ad9bd5d..e99f4fac2cf 100644 --- a/apps/sim/connectors/hubspot/meta.ts +++ b/apps/sim/connectors/hubspot/meta.ts @@ -16,6 +16,8 @@ export const hubspotConnectorMeta: ConnectorMeta = { 'crm.objects.companies.read', 'crm.objects.deals.read', 'tickets', + /** Required by `GET /account-info/v3/details`, used to build record deep links. */ + 'oauth', ], }, diff --git a/apps/sim/connectors/incidentio/incidentio.test.ts b/apps/sim/connectors/incidentio/incidentio.test.ts index a264663450a..5ecfafce8fe 100644 --- a/apps/sim/connectors/incidentio/incidentio.test.ts +++ b/apps/sim/connectors/incidentio/incidentio.test.ts @@ -226,6 +226,45 @@ describe('incidentioConnector.listDocuments', () => { expect(syncContext.listingCapped).toBe(true) }) + it('does not flag the listing capped when the cap lands on an exhausted source', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + incidents: [incidentFixture({ id: 'inc-1' })], + pagination_meta: {}, + }) + ) + + const syncContext: Record = {} + const result = await incidentioConnector.listDocuments( + ACCESS_TOKEN, + { maxIncidents: '1' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('requests only the remaining records on the final capped page', async () => { + mockFetch.mockResolvedValue(jsonResponse({ incidents: [] })) + + await incidentioConnector.listDocuments(ACCESS_TOKEN, { maxIncidents: '7' }, undefined, {}) + + expect(requestUrl().searchParams.get('page_size')).toBe('7') + }) + + it('stops paginating when a page returns no incidents but still echoes a cursor', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ incidents: [], pagination_meta: { after: 'cursor-9' } }) + ) + + const result = await incidentioConnector.listDocuments(ACCESS_TOKEN, {}) + + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + }) + it('throws instead of reporting an empty listing when the API fails', async () => { mockFetch.mockResolvedValue(jsonResponse({ error: 'nope' }, 500)) @@ -281,22 +320,45 @@ describe('incidentioConnector.getDocument', () => { expect(doc?.content).toContain('Checkout is down') }) + it('marks the hash partial when updates could not be fetched, so the next sync retries', async () => { + mockFetch + .mockResolvedValueOnce(jsonResponse({ incident: incidentFixture() })) + .mockResolvedValueOnce(jsonResponse({ error: 'nope' }, 500)) + + const partial = await incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1') + + vi.clearAllMocks() + mockFetch + .mockResolvedValueOnce(jsonResponse({ incident: incidentFixture() })) + .mockResolvedValueOnce(jsonResponse({ incident_updates: [] })) + + const complete = await incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1') + + vi.clearAllMocks() + mockFetch.mockResolvedValue(jsonResponse({ incidents: [incidentFixture()] })) + const listed = await incidentioConnector.listDocuments(ACCESS_TOKEN, {}) + const stubHash = listed.documents[0].contentHash + + expect(complete?.contentHash).toBe(stubHash) + expect(partial?.contentHash).not.toBe(stubHash) + }) + it('returns null for a deleted incident', async () => { mockFetch.mockResolvedValue(jsonResponse({}, 404)) await expect(incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1')).resolves.toBeNull() }) - it('returns null instead of throwing when the API fails', async () => { + it('throws when the API fails so the sync engine records a failed row', async () => { mockFetch.mockResolvedValue(jsonResponse({ error: 'nope' }, 500)) - await expect(incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1')).resolves.toBeNull() + await expect(incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1')).rejects.toThrow('500') }) - it('returns null instead of throwing when fetch rejects', async () => { + it('throws when fetch rejects rather than dropping the incident silently', async () => { mockFetch.mockRejectedValue(new Error('boom')) - await expect(incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1')).resolves.toBeNull() + await expect(incidentioConnector.getDocument(ACCESS_TOKEN, {}, 'inc-1')).rejects.toThrow('boom') }) it('returns null for an empty external id without calling the API', async () => { diff --git a/apps/sim/connectors/incidentio/incidentio.ts b/apps/sim/connectors/incidentio/incidentio.ts index 264343fb319..10f75804e32 100644 --- a/apps/sim/connectors/incidentio/incidentio.ts +++ b/apps/sim/connectors/incidentio/incidentio.ts @@ -368,42 +368,75 @@ function incidentToStub(incident: IncidentioIncident): ExternalDocument { /** * Fetches all status updates for an incident, following the `after` cursor and capping * the total to keep getDocument bounded for very long-running incidents. + * + * `complete` is false when the update listing was cut short by an API failure rather + * than by exhaustion or the deliberate {@link MAX_UPDATES_PER_INCIDENT} cap. Callers use + * it to avoid caching a partially-hydrated document under its final content hash. */ async function fetchIncidentUpdates( accessToken: string, incidentId: string -): Promise { +): Promise<{ updates: IncidentioUpdate[]; complete: boolean }> { const updates: IncidentioUpdate[] = [] let after: string | undefined + let complete = true + let sourceHasMore = false while (updates.length < MAX_UPDATES_PER_INCIDENT) { const url = new URL(`${INCIDENTIO_API_BASE}/v2/incident_updates`) url.searchParams.set('incident_id', incidentId) - url.searchParams.set('page_size', String(PAGE_SIZE)) + url.searchParams.set( + 'page_size', + String(Math.min(PAGE_SIZE, MAX_UPDATES_PER_INCIDENT - updates.length)) + ) if (after) url.searchParams.set('after', after) - const response = await fetchWithRetry(url.toString(), { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + let page: IncidentioUpdate[] + try { + const response = await fetchWithRetry(url.toString(), { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) - if (!response.ok) { - logger.warn('Failed to fetch incident updates', { incidentId, status: response.status }) + if (!response.ok) { + logger.warn('Failed to fetch incident updates', { incidentId, status: response.status }) + complete = false + break + } + + const data = (await response.json()) as IncidentioUpdatesListResponse + page = data.incident_updates ?? [] + after = data.pagination_meta?.after?.trim() || undefined + } catch (error) { + logger.warn('Failed to fetch incident updates', { + incidentId, + error: toError(error).message, + }) + complete = false break } - const data = (await response.json()) as IncidentioUpdatesListResponse - const page = data.incident_updates ?? [] updates.push(...page) - after = data.pagination_meta?.after?.trim() || undefined + /** + * `after` is only a usable cursor when the page returned records — the docs do + * not promise it is omitted on the final page, so an empty page must terminate. + */ if (!after || page.length === 0) break + sourceHasMore = true } - return updates.slice(0, MAX_UPDATES_PER_INCIDENT) + if (sourceHasMore && updates.length >= MAX_UPDATES_PER_INCIDENT) { + logger.warn('Truncated incident updates at the per-incident cap', { + incidentId, + cap: MAX_UPDATES_PER_INCIDENT, + }) + } + + return { updates, complete } } export const incidentioConnector: ConnectorConfig = { @@ -422,8 +455,14 @@ export const incidentioConnector: ConnectorConfig = { typeof sourceConfig.statusCategory === 'string' ? sourceConfig.statusCategory.trim() : '' const mode = typeof sourceConfig.mode === 'string' ? sourceConfig.mode.trim() : '' + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = maxIncidents > 0 ? Math.max(0, maxIncidents - prevFetched) : 0 + const url = new URL(`${INCIDENTIO_API_BASE}/v2/incidents`) - url.searchParams.set('page_size', String(PAGE_SIZE)) + url.searchParams.set( + 'page_size', + String(remaining > 0 ? Math.min(PAGE_SIZE, remaining) : PAGE_SIZE) + ) url.searchParams.set('sort_by', 'created_at_oldest_first') if (cursor) url.searchParams.set('after', cursor) if (lastSyncAt) url.searchParams.set('updated_at[gte]', lastSyncAt.toISOString()) @@ -458,22 +497,33 @@ export const incidentioConnector: ConnectorConfig = { const data = (await response.json()) as IncidentioIncidentsListResponse const incidents = (data.incidents ?? []).filter((incident) => Boolean(incident.id)) - const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let documents = incidents.map(incidentToStub) - if (maxIncidents > 0) { - const remaining = Math.max(0, maxIncidents - prevFetched) - if (documents.length > remaining) { - documents = documents.slice(0, remaining) - } + if (maxIncidents > 0 && documents.length > remaining) { + documents = documents.slice(0, remaining) } const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents - if (hitLimit && syncContext) syncContext.listingCapped = true + /** + * `after` is only a usable next-page cursor when this page actually returned + * incidents. incident.io does not document that it is omitted on the final page, + * so trusting it alone risks re-requesting the same tail until the sync engine + * truncates pagination — which permanently disables deletion reconciliation. + */ const after = data.pagination_meta?.after?.trim() || undefined - const hasMore = !hitLimit && Boolean(after) + const sourceHasMore = Boolean(after) && incidents.length > 0 + + const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents + /** + * Only a cap that actually hides still-listed incidents may suppress deletion + * reconciliation. A cap landing exactly on an exhausted source produced a complete + * listing, and flagging it would strand deleted incidents in the KB forever. + */ + const truncatedByCap = hitLimit && (documents.length < incidents.length || sourceHasMore) + if (truncatedByCap && syncContext) syncContext.listingCapped = true + + const hasMore = !hitLimit && sourceHasMore return { documents, @@ -509,10 +559,20 @@ export const incidentioConnector: ConnectorConfig = { const incident = data.incident if (!incident?.id) return null - const updates = await fetchIncidentUpdates(accessToken, incident.id) + const { updates, complete } = await fetchIncidentUpdates(accessToken, incident.id) const content = formatIncidentContent(incident, updates) if (!content.trim()) return null + /** + * A failed updates fetch yields usable but incomplete content. Storing it under the + * canonical hash would freeze the gap in place until the incident itself changes, so + * a partial marker is appended instead: it never matches the list stub's hash, so the + * next sync re-hydrates and self-heals once incident.io responds. + */ + const contentHash = complete + ? buildContentHash(incident) + : `${buildContentHash(incident)}:partial` + return { externalId: incident.id, title: buildTitle(incident), @@ -520,15 +580,20 @@ export const incidentioConnector: ConnectorConfig = { contentDeferred: false, mimeType: 'text/plain', sourceUrl: buildSourceUrl(incident), - contentHash: buildContentHash(incident), + contentHash, metadata: buildMetadata(incident), } } catch (error) { + /** + * Only the 404/410 above means the incident is genuinely gone. Everything else — + * 429, 5xx, network faults — is rethrown so the sync engine records a failed row + * and keeps the already-indexed incident out of deletion reconciliation. + */ logger.warn('Failed to get incident.io incident', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/incidentio/meta.ts b/apps/sim/connectors/incidentio/meta.ts index c1370f68044..5ec4e56decc 100644 --- a/apps/sim/connectors/incidentio/meta.ts +++ b/apps/sim/connectors/incidentio/meta.ts @@ -45,14 +45,14 @@ export const incidentioConnectorMeta: ConnectorMeta = { required: false, mode: 'advanced', options: [ - { label: 'All', id: '' }, + { label: 'Standard and retrospective (default)', id: '' }, { label: 'Standard (real incidents)', id: 'standard' }, { label: 'Retrospective', id: 'retrospective' }, { label: 'Test', id: 'test' }, { label: 'Tutorial', id: 'tutorial' }, ], description: - 'Only sync incidents of this mode. Use Standard to exclude test/tutorial incidents.', + 'Only sync incidents of this mode. Leaving this unset uses the incident.io default, which covers standard and retrospective incidents and excludes test and tutorial ones.', }, { id: 'maxIncidents', diff --git a/apps/sim/connectors/intercom/intercom.test.ts b/apps/sim/connectors/intercom/intercom.test.ts new file mode 100644 index 00000000000..0e83ed6101f --- /dev/null +++ b/apps/sim/connectors/intercom/intercom.test.ts @@ -0,0 +1,353 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { intercomConnector } from '@/connectors/intercom/intercom' + +const ACCESS_TOKEN = 'test-token' + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function requestUrl(callIndex = 0): URL { + const call = mockFetch.mock.calls[callIndex] + if (!call) throw new Error(`No fetch call at index ${callIndex}`) + return new URL(String(call[0])) +} + +function articleFixture(id: string, overrides: Record = {}) { + return { + type: 'article', + id, + title: `Article ${id}`, + description: null, + body: `

Body ${id}

`, + author_id: 5017691, + state: 'published', + created_at: 1672928359, + updated_at: 1672928610, + url: `https://help.example.com/articles/${id}`, + ...overrides, + } +} + +function conversationFixture(id: string, overrides: Record = {}) { + return { + type: 'conversation', + id, + title: `Conversation ${id}`, + state: 'open', + created_at: 1663597223, + updated_at: 1663597260, + source: { + type: 'conversation', + id: '403918241', + body: '

hello

', + author: { type: 'admin', id: '991', name: 'Ada' }, + }, + tags: { type: 'tag.list', tags: [{ id: '1', name: 'billing' }] }, + ...overrides, + } +} + +beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('intercom regional base URL', () => { + it.each([ + ['us', 'api.intercom.io'], + ['eu', 'api.eu.intercom.io'], + ['au', 'api.au.intercom.io'], + [undefined, 'api.intercom.io'], + ])('routes region %s to %s', async (region, host) => { + mockFetch.mockResolvedValue(jsonResponse({ data: [], pages: { total_pages: 0 } })) + + await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', ...(region ? { region } : {}) }, + undefined, + {} + ) + + expect(requestUrl().host).toBe(host) + }) + + it('sends the pinned Intercom-Version header', async () => { + mockFetch.mockResolvedValue(jsonResponse({ data: [], pages: { total_pages: 0 } })) + + await intercomConnector.listDocuments(ACCESS_TOKEN, { contentType: 'articles' }, undefined, {}) + + const init = mockFetch.mock.calls[0][1] as RequestInit + expect((init.headers as Record)['Intercom-Version']).toBe('2.11') + }) +}) + +describe('intercom listingCapped', () => { + it('leaves listingCapped unset when the source is exhausted', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ data: [articleFixture('1'), articleFixture('2')], pages: { total_pages: 1 } }) + ) + const syncContext: Record = {} + + const result = await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', maxItems: '10' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('leaves listingCapped unset when maxItems lands exactly on exhaustion', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ data: [articleFixture('1'), articleFixture('2')], pages: { total_pages: 1 } }) + ) + const syncContext: Record = {} + + await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', maxItems: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxItems hides articles on the same page', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + data: [articleFixture('1'), articleFixture('2'), articleFixture('3')], + pages: { total_pages: 1 }, + }) + ) + const syncContext: Record = {} + + await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', maxItems: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when conversations remain behind a cursor', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + type: 'conversation.list', + conversations: [conversationFixture('a'), conversationFixture('b')], + pages: { type: 'pages', next: { starting_after: 'cursor-1' } }, + }) + ) + const syncContext: Record = {} + + await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'conversations', maxItems: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('does not flag listingCapped for an intentional state filter', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + data: [articleFixture('1'), articleFixture('2', { state: 'draft' })], + pages: { total_pages: 1 }, + }) + ) + const syncContext: Record = {} + + const result = await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', articleState: 'published', maxItems: '10' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBeUndefined() + }) +}) + +describe('intercom articles pagination', () => { + it('keeps per_page constant across pages so page-number paging cannot slide', async () => { + const fullPage = Array.from({ length: 50 }, (_, i) => articleFixture(String(i + 1))) + mockFetch.mockImplementation(() => + Promise.resolve(jsonResponse({ data: fullPage, pages: { total_pages: 3 } })) + ) + + await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles', maxItems: '60' }, + undefined, + {} + ) + + expect(mockFetch.mock.calls).toHaveLength(2) + expect(requestUrl(0).searchParams.get('per_page')).toBe('50') + expect(requestUrl(0).searchParams.get('page')).toBe('1') + expect(requestUrl(1).searchParams.get('per_page')).toBe('50') + expect(requestUrl(1).searchParams.get('page')).toBe('2') + }) +}) + +describe('intercom maxItems budget', () => { + it('shares one budget across articles and conversations', async () => { + mockFetch.mockImplementation((url: string) => { + if (String(url).includes('/articles')) { + return Promise.resolve( + jsonResponse({ + data: [articleFixture('1'), articleFixture('2')], + pages: { total_pages: 1 }, + }) + ) + } + return Promise.resolve( + jsonResponse({ conversations: [conversationFixture('a')], pages: { type: 'pages' } }) + ) + }) + const syncContext: Record = {} + + const result = await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'both', maxItems: '3' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(3) + const conversationCall = mockFetch.mock.calls.find((c) => + String(c[0]).includes('/conversations') + ) + expect(new URL(String(conversationCall?.[0])).searchParams.get('per_page')).toBe('1') + }) +}) + +describe('intercom contentHash consistency', () => { + it('matches between the conversation stub and getDocument', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + conversations: [conversationFixture('a')], + pages: { type: 'pages' }, + }) + ) + const listed = await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'conversations' }, + undefined, + {} + ) + const stub = listed.documents[0] + expect(stub.contentDeferred).toBe(true) + expect(stub.content).toBe('') + + mockFetch.mockResolvedValue( + jsonResponse({ + ...conversationFixture('a'), + conversation_parts: { + type: 'conversation_part.list', + conversation_parts: [ + { + type: 'conversation_part', + id: '3', + part_type: 'comment', + body: '

Okay!

', + created_at: 1663597240, + author: { type: 'admin', id: '991', name: 'Ada' }, + }, + ], + total_count: 1, + }, + }) + ) + + const full = await intercomConnector.getDocument( + ACCESS_TOKEN, + { contentType: 'conversations' }, + stub.externalId + ) + + expect(full?.contentHash).toBe(stub.contentHash) + expect(full?.contentDeferred).toBe(false) + expect(full?.content).toContain('Okay!') + expect(full?.content).not.toContain('

') + expect(full?.metadata?.messageCount).toBe(2) + }) +}) + +describe('intercom getDocument error handling', () => { + it('returns null on 404', async () => { + mockFetch.mockResolvedValue(jsonResponse({ errors: [{ code: 'not_found' }] }, 404)) + + await expect( + intercomConnector.getDocument(ACCESS_TOKEN, {}, 'conversation-missing') + ).resolves.toBeNull() + }) + + it('rethrows non-404 failures so the sync engine records them', async () => { + mockFetch.mockResolvedValue(jsonResponse({ errors: [{ code: 'server_error' }] }, 500)) + + await expect(intercomConnector.getDocument(ACCESS_TOKEN, {}, 'conversation-1')).rejects.toThrow( + /500/ + ) + }) +}) + +describe('intercom mapTags', () => { + it('only emits tag ids declared in tagDefinitions', async () => { + const declared = new Set((intercomConnector.tagDefinitions ?? []).map((t) => t.id)) + const mapped = intercomConnector.mapTags?.({ + type: 'conversation', + state: 'open', + tags: 'billing', + authorId: '5017691', + messageCount: 3, + updatedAt: '2026-01-01T00:00:00.000Z', + }) + + for (const key of Object.keys(mapped ?? {})) { + expect(declared.has(key)).toBe(true) + } + expect(mapped?.updatedAt).toBeInstanceOf(Date) + }) + + it('omits authorId when the article has none', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + data: [articleFixture('1', { author_id: null })], + pages: { total_pages: 1 }, + }) + ) + + const result = await intercomConnector.listDocuments( + ACCESS_TOKEN, + { contentType: 'articles' }, + undefined, + {} + ) + + expect(result.documents[0].metadata?.authorId).toBeUndefined() + expect(intercomConnector.mapTags?.(result.documents[0].metadata ?? {})).not.toHaveProperty( + 'authorId' + ) + }) +}) diff --git a/apps/sim/connectors/intercom/intercom.ts b/apps/sim/connectors/intercom/intercom.ts index 7a12d1035df..2fba2a0dc52 100644 --- a/apps/sim/connectors/intercom/intercom.ts +++ b/apps/sim/connectors/intercom/intercom.ts @@ -2,38 +2,63 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' -import { DEFAULT_MAX_ITEMS, intercomConnectorMeta } from '@/connectors/intercom/meta' +import { + DEFAULT_INTERCOM_REGION, + DEFAULT_MAX_ITEMS, + INTERCOM_API_BASE_BY_REGION, + intercomConnectorMeta, +} from '@/connectors/intercom/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' const logger = createLogger('IntercomConnector') -const INTERCOM_API_BASE = 'https://api.intercom.io' +/** + * Intercom pins request/response shapes to an `Intercom-Version` header. 2.11 is a + * supported version (Intercom supports 2.7+) and is the version whose documented + * schemas this connector parses. + * + * @see https://developers.intercom.com/docs/build-an-integration/learn-more/rest-apis/api-versioning + */ +const INTERCOM_API_VERSION = '2.11' + +/** + * `/articles` is page-number paginated (`page` selects the n-th chunk of `per_page` + * records), so `per_page` must stay constant across pages or a later page would + * slide backwards and skip articles. Intercom caps `per_page` at 150 API-wide. + */ const ARTICLES_PER_PAGE = 50 -const CONVERSATIONS_PER_PAGE = 50 +/** Intercom caps `per_page` at 150 across every list API. */ +const CONVERSATIONS_PER_PAGE = 150 + +/** Intercom returns at most 500 conversation parts per conversation. */ +const MAX_CONVERSATION_PARTS = 500 +const CONTENT_TYPES = new Set(['articles', 'conversations', 'both']) + +/** + * Every schema here models only the fields this connector reads. `.passthrough()` + * carries the rest of Intercom's payload through untouched, so unmodeled or newly + * added fields never fail a parse. + */ const IntercomAuthorSchema = z .object({ - type: z.string(), - id: z.string(), - name: z.string().optional(), + type: z.string().optional(), + name: z.string().nullable().optional(), }) .passthrough() const IntercomArticleSchema = z .object({ - type: z.string().optional(), id: z.string(), - title: z.string().optional(), + title: z.string().nullable().optional(), description: z.string().nullable().optional(), body: z.string().nullable().optional(), - author_id: z.union([z.number(), z.string()]).optional(), + author_id: z.union([z.number(), z.string()]).nullable().optional(), state: z.string(), created_at: z.number(), updated_at: z.number(), - url: z.string().optional(), - parent_id: z.number().nullable().optional(), - parent_type: z.string().nullable().optional(), + url: z.string().nullable().optional(), }) .passthrough() @@ -41,9 +66,6 @@ type IntercomArticle = z.infer const IntercomConversationPartSchema = z .object({ - type: z.string().optional(), - id: z.string(), - part_type: z.string().optional(), body: z.string().nullable().optional(), created_at: z.number(), author: IntercomAuthorSchema.optional(), @@ -52,45 +74,29 @@ const IntercomConversationPartSchema = z const IntercomConversationSchema = z .object({ - type: z.string().optional(), id: z.string(), created_at: z.number(), updated_at: z.number(), title: z.string().nullable().optional(), state: z.string(), - open: z.boolean().optional(), source: z .object({ - type: z.string(), - id: z.string(), - subject: z.string().optional(), body: z.string().nullable().optional(), - author: IntercomAuthorSchema, - delivered_as: z.string().optional(), + author: IntercomAuthorSchema.optional(), }) .passthrough() .optional(), tags: z .object({ - type: z.string().optional(), - tags: z - .array( - z - .object({ - id: z.string(), - name: z.string(), - }) - .passthrough() - ) - .default([]), + tags: z.array(z.object({ name: z.string() }).passthrough()).default([]), }) .passthrough() .optional(), + /** Omitted entirely by `GET /conversations`; only the single-conversation fetch returns it. */ conversation_parts: z .object({ - type: z.string(), - conversation_parts: z.array(IntercomConversationPartSchema), - total_count: z.number(), + conversation_parts: z.array(IntercomConversationPartSchema).default([]), + total_count: z.number().optional(), }) .passthrough() .optional(), @@ -99,16 +105,41 @@ const IntercomConversationSchema = z type IntercomConversation = z.infer +/** HTTP failure from the Intercom API, carrying the status so callers can branch on 404. */ +class IntercomHttpError extends Error { + constructor( + readonly status: number, + message: string + ) { + super(message) + this.name = 'IntercomHttpError' + } +} + +/** + * Resolves the regional REST host for the workspace. Intercom serves EU and AU + * workspaces from dedicated hosts; an unset/unknown region falls back to US. + */ +function resolveApiBase(sourceConfig: Record): string { + const region = + typeof sourceConfig.region === 'string' ? sourceConfig.region.trim().toLowerCase() : '' + return ( + INTERCOM_API_BASE_BY_REGION[region as keyof typeof INTERCOM_API_BASE_BY_REGION] ?? + INTERCOM_API_BASE_BY_REGION[DEFAULT_INTERCOM_REGION] + ) +} + /** * Makes a GET request to the Intercom API with Bearer token auth. */ async function intercomApiGet( + apiBase: string, path: string, accessToken: string, params?: Record, retryOptions?: Parameters[2] ): Promise> { - const url = new URL(`${INTERCOM_API_BASE}${path}`) + const url = new URL(`${apiBase}${path}`) if (params) { for (const [key, value] of Object.entries(params)) { url.searchParams.set(key, value) @@ -122,7 +153,7 @@ async function intercomApiGet( headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', - 'Intercom-Version': '2.11', + 'Intercom-Version': INTERCOM_API_VERSION, }, }, retryOptions @@ -130,25 +161,34 @@ async function intercomApiGet( if (!response.ok) { const errorBody = await response.text().catch(() => '') - throw new Error(`Intercom API HTTP error ${response.status}: ${errorBody}`) + throw new IntercomHttpError( + response.status, + `Intercom API HTTP error ${response.status}: ${errorBody}` + ) } return (await response.json()) as Record } /** - * Fetches all articles from Intercom, respecting page-based pagination and max items cap. + * Fetches articles using Intercom's page-based `/articles` pagination. + * + * `capped` reports whether the `maxItems` budget hid articles that still exist in + * the source. A budget landing exactly on source exhaustion is NOT capped — the + * listing is complete and deletion reconciliation must stay enabled. */ async function fetchArticles( + apiBase: string, accessToken: string, maxItems: number, stateFilter: string -): Promise { - const allArticles: IntercomArticle[] = [] +): Promise<{ articles: IntercomArticle[]; capped: boolean }> { + const collected: IntercomArticle[] = [] let page = 1 + let capped = false - while (allArticles.length < maxItems) { - const data = await intercomApiGet('/articles', accessToken, { + while (collected.length < maxItems) { + const data = await intercomApiGet(apiBase, '/articles', accessToken, { page: String(page), per_page: String(ARTICLES_PER_PAGE), }) @@ -156,67 +196,84 @@ async function fetchArticles( const articles = z.array(IntercomArticleSchema).parse(data.data ?? []) if (articles.length === 0) break + let consumed = 0 for (const article of articles) { + consumed++ if (stateFilter !== 'all' && article.state !== stateFilter) continue - allArticles.push(article) - if (allArticles.length >= maxItems) break + collected.push(article) + if (collected.length >= maxItems) break } const pages = data.pages as { total_pages?: number } | null - if (!pages?.total_pages || page >= pages.total_pages) break + const morePages = Boolean(pages?.total_pages && page < pages.total_pages) + + if (collected.length >= maxItems) { + capped = consumed < articles.length || morePages + break + } + if (!morePages) break page++ } - return allArticles + return { articles: collected, capped } } /** - * Fetches conversations from Intercom using cursor-based pagination. + * Fetches conversations using Intercom's cursor-based `/conversations` pagination + * (`pages.next.starting_after`). + * + * See {@link fetchArticles} for the `capped` contract. */ async function fetchConversations( + apiBase: string, accessToken: string, maxItems: number, stateFilter: string -): Promise { - const allConversations: IntercomConversation[] = [] +): Promise<{ conversations: IntercomConversation[]; capped: boolean }> { + const collected: IntercomConversation[] = [] let startingAfter: string | undefined - - while (allConversations.length < maxItems) { - const params: Record = { - per_page: String(Math.min(CONVERSATIONS_PER_PAGE, 150)), - } + let capped = false + + while (collected.length < maxItems) { + /** + * `/conversations` is cursor paginated, so shrinking the last page is safe. + * A state filter drops rows client-side, so only an unfiltered listing can + * size the page from the remaining budget. + */ + const perPage = + stateFilter === 'all' + ? Math.min(CONVERSATIONS_PER_PAGE, maxItems - collected.length) + : CONVERSATIONS_PER_PAGE + + const params: Record = { per_page: String(perPage) } if (startingAfter) { params.starting_after = startingAfter } - const data = await intercomApiGet('/conversations', accessToken, params) + const data = await intercomApiGet(apiBase, '/conversations', accessToken, params) const conversations = z.array(IntercomConversationSchema).parse(data.conversations ?? []) if (conversations.length === 0) break + let consumed = 0 for (const conversation of conversations) { + consumed++ if (stateFilter !== 'all' && conversation.state !== stateFilter) continue - allConversations.push(conversation) - if (allConversations.length >= maxItems) break + collected.push(conversation) + if (collected.length >= maxItems) break } - const pages = data.pages as { next?: { starting_after?: string } } | null - const nextCursor = pages?.next?.starting_after + const pages = data.pages as { next?: { starting_after?: string | null } | null } | null + const nextCursor = pages?.next?.starting_after || undefined + + if (collected.length >= maxItems) { + capped = consumed < conversations.length || Boolean(nextCursor) + break + } if (!nextCursor) break startingAfter = nextCursor } - return allConversations -} - -/** - * Fetches the full conversation with conversation_parts included. - */ -async function fetchConversationDetail( - accessToken: string, - conversationId: string -): Promise { - const data = await intercomApiGet(`/conversations/${conversationId}`, accessToken) - return IntercomConversationSchema.parse(data) + return { conversations: collected, capped } } /** @@ -265,6 +322,64 @@ function formatArticle(article: IntercomArticle): string { return parts.join('\n\n') } +function articleMetadata(article: IntercomArticle): Record { + const metadata: Record = { + type: 'article', + state: article.state, + updatedAt: new Date(article.updated_at * 1000).toISOString(), + createdAt: new Date(article.created_at * 1000).toISOString(), + } + if (article.author_id !== undefined && article.author_id !== null) { + metadata.authorId = String(article.author_id) + } + return metadata +} + +/** + * Shared article projection so `listDocuments` and `getDocument` cannot drift on + * `externalId`, `contentHash`, or `sourceUrl`. + */ +function articleToDocument(article: IntercomArticle): ExternalDocument { + return { + externalId: `article-${article.id}`, + title: article.title || `Article ${article.id}`, + content: formatArticle(article), + mimeType: 'text/plain', + sourceUrl: + article.url || `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`, + contentHash: `intercom:article-${article.id}:${article.updated_at}`, + metadata: articleMetadata(article), + } +} + +function conversationMetadata(conversation: IntercomConversation): Record { + return { + type: 'conversation', + state: conversation.state, + tags: (conversation.tags?.tags?.map((t) => t.name) || []).join(', '), + updatedAt: new Date(conversation.updated_at * 1000).toISOString(), + createdAt: new Date(conversation.created_at * 1000).toISOString(), + } +} + +/** + * Lightweight listing stub. `/conversations` never returns `conversation_parts` + * (documented as "ignored when Listing all Conversations"), so the transcript is + * hydrated per-document via `getDocument`. + */ +function conversationToStub(conversation: IntercomConversation): ExternalDocument { + return { + externalId: `conversation-${conversation.id}`, + title: conversation.title || `Conversation #${conversation.id}`, + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${conversation.id}`, + contentHash: `intercom:conversation-${conversation.id}:${conversation.updated_at}`, + metadata: conversationMetadata(conversation), + } +} + export const intercomConnector: ConnectorConfig = { ...intercomConnectorMeta, @@ -272,72 +387,69 @@ export const intercomConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { + const apiBase = resolveApiBase(sourceConfig) const contentType = (sourceConfig.contentType as string) || 'articles' const articleState = (sourceConfig.articleState as string) || 'published' const conversationState = (sourceConfig.conversationState as string) || 'all' - const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS + const parsedMax = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS + const maxItems = + Number.isFinite(parsedMax) && parsedMax > 0 ? Math.floor(parsedMax) : DEFAULT_MAX_ITEMS const documents: ExternalDocument[] = [] + let capped = false if (contentType === 'articles' || contentType === 'both') { logger.info('Fetching Intercom articles', { articleState, maxItems }) - const articles = await fetchArticles(accessToken, maxItems, articleState) + const result = await fetchArticles(apiBase, accessToken, maxItems, articleState) + capped = capped || result.capped - for (const article of articles) { - const content = formatArticle(article) - if (!content.trim()) continue - - const updatedAt = new Date(article.updated_at * 1000).toISOString() - - documents.push({ - externalId: `article-${article.id}`, - title: article.title || `Article ${article.id}`, - content, - mimeType: 'text/plain', - sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`, - contentHash: `intercom:article-${article.id}:${article.updated_at}`, - metadata: { - type: 'article', - state: article.state, - authorId: String(article.author_id), - updatedAt, - createdAt: new Date(article.created_at * 1000).toISOString(), - }, - }) + for (const article of result.articles) { + const doc = articleToDocument(article) + if (!doc.content.trim()) continue + documents.push(doc) } - logger.info('Fetched Intercom articles', { count: articles.length }) + logger.info('Fetched Intercom articles', { count: result.articles.length }) } - if (contentType === 'conversations' || contentType === 'both') { + /** + * `maxItems` is a single budget for the whole sync, so an "articles & + * conversations" source cannot silently fetch 2x the configured cap. + */ + const conversationBudget = maxItems - documents.length + + if ((contentType === 'conversations' || contentType === 'both') && conversationBudget > 0) { logger.info('Fetching Intercom conversations', { conversationState, maxItems }) - const conversations = await fetchConversations(accessToken, maxItems, conversationState) - - for (const conversation of conversations) { - const updatedAt = new Date(conversation.updated_at * 1000).toISOString() - const tags = conversation.tags?.tags?.map((t) => t.name) || [] - - documents.push({ - externalId: `conversation-${conversation.id}`, - title: conversation.title || `Conversation #${conversation.id}`, - content: '', - contentDeferred: true, - mimeType: 'text/plain', - sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${conversation.id}`, - contentHash: `intercom:conversation-${conversation.id}:${conversation.updated_at}`, - metadata: { - type: 'conversation', - state: conversation.state, - tags: tags.join(', '), - updatedAt, - createdAt: new Date(conversation.created_at * 1000).toISOString(), - }, - }) + const result = await fetchConversations( + apiBase, + accessToken, + conversationBudget, + conversationState + ) + capped = capped || result.capped + + for (const conversation of result.conversations) { + documents.push(conversationToStub(conversation)) } - logger.info('Fetched Intercom conversations', { count: conversations.length }) + logger.info('Fetched Intercom conversations', { count: result.conversations.length }) + } else if (contentType === 'both' && conversationBudget <= 0) { + capped = true + } + + /** + * The sync engine hard-deletes stored documents absent from a full listing. + * Only flag when the cap actually hid still-existing items — flagging a + * complete listing would permanently block deletion reconciliation. + */ + if (capped && syncContext) { + syncContext.listingCapped = true + logger.warn('Intercom listing truncated by maxItems; skipping deletion reconciliation', { + maxItems, + docsListed: documents.length, + }) } return { documents, hasMore: false } @@ -348,59 +460,54 @@ export const intercomConnector: ConnectorConfig = { sourceConfig: Record, externalId: string ): Promise => { + const apiBase = resolveApiBase(sourceConfig) + try { if (externalId.startsWith('article-')) { - const articleId = externalId.replace('article-', '') - const data = await intercomApiGet(`/articles/${articleId}`, accessToken) + const articleId = externalId.slice('article-'.length) + const data = await intercomApiGet( + apiBase, + `/articles/${encodeURIComponent(articleId)}`, + accessToken + ) const article = IntercomArticleSchema.parse(data) - const content = formatArticle(article) - if (!content.trim()) return null - - const updatedAt = new Date(article.updated_at * 1000).toISOString() - - return { - externalId, - title: article.title || `Article ${article.id}`, - content, - mimeType: 'text/plain', - sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`, - contentHash: `intercom:article-${article.id}:${article.updated_at}`, - metadata: { - type: 'article', - state: article.state, - authorId: String(article.author_id), - updatedAt, - createdAt: new Date(article.created_at * 1000).toISOString(), - }, - } + const doc = articleToDocument(article) + if (!doc.content.trim()) return null + return { ...doc, externalId } } if (externalId.startsWith('conversation-')) { - const conversationId = externalId.replace('conversation-', '') - const detail = await fetchConversationDetail(accessToken, conversationId) + const conversationId = externalId.slice('conversation-'.length) + const data = await intercomApiGet( + apiBase, + `/conversations/${encodeURIComponent(conversationId)}`, + accessToken + ) + const detail = IntercomConversationSchema.parse(data) const content = formatConversation(detail) if (!content.trim()) return null - const updatedAt = new Date(detail.updated_at * 1000).toISOString() - const tags = detail.tags?.tags?.map((t) => t.name) || [] + const returnedParts = detail.conversation_parts?.conversation_parts?.length ?? 0 + const totalParts = detail.conversation_parts?.total_count ?? returnedParts + if (totalParts > returnedParts || returnedParts >= MAX_CONVERSATION_PARTS) { + logger.warn('Intercom returned a truncated conversation transcript', { + externalId, + returnedParts, + totalParts, + limit: MAX_CONVERSATION_PARTS, + }) + } return { + ...conversationToStub(detail), externalId, - title: detail.title || `Conversation #${detail.id}`, content, contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${detail.id}`, - contentHash: `intercom:conversation-${detail.id}:${detail.updated_at}`, metadata: { - type: 'conversation', - state: detail.state, - tags: tags.join(', '), - updatedAt, - createdAt: new Date(detail.created_at * 1000).toISOString(), - messageCount: (detail.conversation_parts?.total_count ?? 0) + 1, + ...conversationMetadata(detail), + messageCount: totalParts + 1, }, } } @@ -408,11 +515,20 @@ export const intercomConnector: ConnectorConfig = { logger.warn('Unknown external ID format', { externalId }) return null } catch (error) { - logger.warn('Failed to get Intercom document', { + /** + * Only a genuine 404 means "gone". Anything else is a transient/unexpected + * failure and must propagate so the sync engine records a failed document + * instead of silently dropping it from this run. + */ + if (error instanceof IntercomHttpError && error.status === 404) { + logger.warn('Intercom document not found', { externalId }) + return null + } + logger.error('Failed to get Intercom document', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, @@ -427,14 +543,24 @@ export const intercomConnector: ConnectorConfig = { return { valid: false, error: 'Content type is required' } } + if (!CONTENT_TYPES.has(contentType)) { + return { + valid: false, + error: 'Content type must be one of: articles, conversations, both', + } + } + if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) { return { valid: false, error: 'Max items must be a positive number' } } + const apiBase = resolveApiBase(sourceConfig) + try { // Verify API access by fetching the first page of articles or conversations if (contentType === 'articles' || contentType === 'both') { await intercomApiGet( + apiBase, '/articles', accessToken, { page: '1', per_page: '1' }, @@ -444,6 +570,7 @@ export const intercomConnector: ConnectorConfig = { if (contentType === 'conversations' || contentType === 'both') { await intercomApiGet( + apiBase, '/conversations', accessToken, { per_page: '1' }, @@ -477,7 +604,7 @@ export const intercomConnector: ConnectorConfig = { result.authorId = metadata.authorId } - if (typeof metadata.messageCount === 'number') { + if (typeof metadata.messageCount === 'number' && Number.isFinite(metadata.messageCount)) { result.messageCount = metadata.messageCount } diff --git a/apps/sim/connectors/intercom/meta.ts b/apps/sim/connectors/intercom/meta.ts index 124ca6120ba..a83c6659328 100644 --- a/apps/sim/connectors/intercom/meta.ts +++ b/apps/sim/connectors/intercom/meta.ts @@ -3,11 +3,28 @@ import type { ConnectorMeta } from '@/connectors/types' export const DEFAULT_MAX_ITEMS = 500 +/** + * Intercom serves regionally-hosted workspaces from dedicated hosts. A token + * issued for an EU or AU workspace is only guaranteed to resolve against its own + * regional host, so the region is part of the connector's configuration. + * + * @see https://developers.intercom.com/docs/build-an-integration/learn-more/rest-apis + */ +export const INTERCOM_API_BASE_BY_REGION = { + us: 'https://api.intercom.io', + eu: 'https://api.eu.intercom.io', + au: 'https://api.au.intercom.io', +} as const + +export type IntercomRegion = keyof typeof INTERCOM_API_BASE_BY_REGION + +export const DEFAULT_INTERCOM_REGION: IntercomRegion = 'us' + export const intercomConnectorMeta: ConnectorMeta = { id: 'intercom', name: 'Intercom', description: 'Sync Help Center articles and conversations from Intercom', - version: '1.0.0', + version: '1.1.0', icon: IntercomIcon, auth: { @@ -29,6 +46,18 @@ export const intercomConnectorMeta: ConnectorMeta = { { label: 'Articles & Conversations', id: 'both' }, ], }, + { + id: 'region', + title: 'Data Region', + type: 'dropdown', + required: false, + description: 'Regional data hosting for your Intercom workspace (default: US)', + options: [ + { label: 'US (api.intercom.io)', id: 'us' }, + { label: 'Europe (api.eu.intercom.io)', id: 'eu' }, + { label: 'Australia (api.au.intercom.io)', id: 'au' }, + ], + }, { id: 'articleState', title: 'Article State', @@ -50,6 +79,7 @@ export const intercomConnectorMeta: ConnectorMeta = { options: [ { label: 'Open', id: 'open' }, { label: 'Closed', id: 'closed' }, + { label: 'Snoozed', id: 'snoozed' }, { label: 'All', id: 'all' }, ], }, @@ -59,7 +89,7 @@ export const intercomConnectorMeta: ConnectorMeta = { type: 'short-input', required: false, placeholder: `e.g. 200 (default: ${DEFAULT_MAX_ITEMS})`, - description: 'Maximum number of articles or conversations to sync', + description: 'Maximum total number of articles and conversations to sync', }, ], diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index a0412873169..a957b3f7468 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jiraConnectorMeta } from '@/connectors/jira/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -8,7 +9,17 @@ import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' const logger = createLogger('JiraConnector') -const PAGE_SIZE = 50 +/** + * `maxResults` on `/rest/api/3/search/jql` is a ceiling, not a guarantee: the + * docs state the API "may return fewer items per page where a large number of + * fields or properties are requested", and that the documented 5000 maximum is + * only reached "when requesting `id` or `key` only". Since this listing requests + * a field selection, a request for more than ~100 buys nothing. + * + * Under-delivery is harmless either way — end-of-results is signalled purely by + * the absence of `nextPageToken`, never by a short page. + */ +const PAGE_SIZE = 100 /** * Builds a JQL clause restricting issues to the given project keys. @@ -36,12 +47,26 @@ function buildIssueContent(fields: Record): string { const description = extractAdfText(fields.description) if (description) parts.push(description) - const comments = fields.comment as { comments?: Array<{ body?: unknown }> } | undefined + const comments = fields.comment as + | { comments?: Array<{ body?: unknown }>; total?: number } + | undefined if (comments?.comments) { for (const comment of comments.comments) { const text = extractAdfText(comment.body) if (text) parts.push(text) } + /** + * The `comment` field on `GET /rest/api/3/issue/{id}` is a paginated + * container: it reports `total` alongside the subset it actually inlines. No + * exact inline limit is documented, so the only reliable signal that an issue + * was indexed without part of its thread is `total` exceeding what arrived. + */ + if (typeof comments.total === 'number' && comments.total > comments.comments.length) { + logger.warn('Jira issue comments truncated by the API; indexing the returned subset', { + returned: comments.comments.length, + total: comments.total, + }) + } } return parts.join('\n\n').trim() @@ -52,7 +77,7 @@ function buildIssueContent(fields: Record): string { * stub with deferred content. The contentHash is metadata-based so it is * identical whether produced during listing or full fetch. */ -function issueToStub(issue: Record, domain: string): ExternalDocument { +function issueToStub(issue: Record, siteUrl: string): ExternalDocument { const fields = (issue.fields || {}) as Record const key = issue.key as string const issueType = fields.issuetype as Record | undefined @@ -70,7 +95,7 @@ function issueToStub(issue: Record, domain: string): ExternalDo content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: `https://${domain}/browse/${key}`, + sourceUrl: `${siteUrl}/browse/${key}`, contentHash: `jira:${issue.id}:${updated}`, metadata: { key, @@ -91,8 +116,8 @@ function issueToStub(issue: Record, domain: string): ExternalDo * Converts a fully-fetched Jira issue (with description and comments) into an * ExternalDocument with resolved content. */ -function issueToFullDocument(issue: Record, domain: string): ExternalDocument { - const stub = issueToStub(issue, domain) +function issueToFullDocument(issue: Record, siteUrl: string): ExternalDocument { + const stub = issueToStub(issue, siteUrl) const fields = (issue.fields || {}) as Record const content = buildIssueContent(fields) @@ -113,6 +138,7 @@ export const jiraConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const domain = sourceConfig.domain as string + const siteUrl = normalizeAtlassianSiteUrl(domain) const projectKeys = parseMultiValue(sourceConfig.projectKey) const jqlFilter = (sourceConfig.jql as string) || '' const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0 @@ -154,6 +180,12 @@ export const jiraConnector: ConnectorConfig = { const remaining = maxIssues > 0 ? Math.max(0, maxIssues - collectedSoFar) : PAGE_SIZE if (maxIssues > 0 && remaining === 0) { + /** + * The cap was already exhausted by an earlier page, so this listing is a + * strict subset of the source. Flag it so the sync engine does not treat + * the missing issues as deletions. + */ + if (syncContext) syncContext.listingCapped = true return { documents: [], hasMore: false } } @@ -193,19 +225,35 @@ export const jiraConnector: ConnectorConfig = { const data = await response.json() let issues = (data.issues || []) as Record[] /** - * `/rest/api/3/search/jql` signals end-of-results purely by the absence - * of `nextPageToken`. `data.isLast` is unreliable on this endpoint and - * has been observed returning `true` alongside a valid token - * (JRACLOUD-95477), so we ignore it. + * `/rest/api/3/search/jql` signals end-of-results purely by the absence of + * `nextPageToken` — the parameter docs state the field "is **not included** + * in the response for the last page". `data.isLast` carries the same intent + * but is redundant, so the token is the single source of truth here. */ const nextPageToken = data.nextPageToken as string | undefined const isLast = !nextPageToken + let slicedByCap = false if (maxIssues > 0 && issues.length > remaining) { issues = issues.slice(0, remaining) + slicedByCap = true + } + + /** + * `warnings` is documented as covering the cases where the server itself + * degraded the result set — "when a JQL clause exceeded its argument limit + * or when the result set was truncated due to an ingestion limit" — so any + * warning means this page is not a faithful view of the source. The field is + * flagged Experimental and "may be absent, empty, or change shape without + * notice", so only its presence is relied on, never its contents. + */ + const warnings = data.warnings as Array<{ type?: string; message?: string }> | undefined + const serverDegradedResults = Boolean(warnings?.length) + if (serverDegradedResults) { + logger.warn('Jira search returned warnings; skipping deletion reconciliation', { warnings }) } - const documents: ExternalDocument[] = issues.map((issue) => issueToStub(issue, domain)) + const documents: ExternalDocument[] = issues.map((issue) => issueToStub(issue, siteUrl)) const newCollected = collectedSoFar + issues.length if (syncContext) syncContext.collectedCount = newCollected @@ -213,6 +261,18 @@ export const jiraConnector: ConnectorConfig = { const reachedCap = maxIssues > 0 && newCollected >= maxIssues const hasMore = !isLast && !reachedCap + /** + * The sync engine hard-deletes stored documents absent from a complete + * listing, so a `maxIssues` cap that truncated the source set must suppress + * reconciliation. Only flag when issues actually remain unlisted — a cap + * that happens to land exactly on the last page is genuine exhaustion and + * must still reconcile deletions. The user-supplied JQL filter is an + * intentional scope narrowing and deliberately does NOT flag. + */ + if ((slicedByCap || (reachedCap && !isLast) || serverDegradedResults) && syncContext) { + syncContext.listingCapped = true + } + return { documents, nextCursor: hasMore && nextPageToken ? `${nextPageToken}|${newCollected}` : undefined, @@ -227,6 +287,7 @@ export const jiraConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const domain = sourceConfig.domain as string + const siteUrl = normalizeAtlassianSiteUrl(domain) let cloudId = syncContext?.cloudId as string | undefined if (!cloudId) { cloudId = await getJiraCloudId(domain, accessToken) @@ -239,7 +300,7 @@ export const jiraConnector: ConnectorConfig = { 'summary,description,comment,issuetype,status,priority,assignee,reporter,project,labels,created,updated' ) - const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}` + const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${encodeURIComponent(externalId)}?${params.toString()}` const response = await fetchWithRetry(url, { method: 'GET', @@ -255,7 +316,7 @@ export const jiraConnector: ConnectorConfig = { } const issue = await response.json() - return issueToFullDocument(issue, domain) + return issueToFullDocument(issue, siteUrl) }, validateConfig: async ( @@ -302,7 +363,7 @@ export const jiraConnector: ConnectorConfig = { if (response.status === 400) { return { valid: false, - error: `One or more projects not found (${projectKeys.join(', ')}) or JQL is invalid`, + error: `One or more projects not found or not accessible: ${projectKeys.join(', ')}`, } } return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` } diff --git a/apps/sim/connectors/jsm/jsm.ts b/apps/sim/connectors/jsm/jsm.ts index be5e1b72184..ee42607ed9a 100644 --- a/apps/sim/connectors/jsm/jsm.ts +++ b/apps/sim/connectors/jsm/jsm.ts @@ -3,7 +3,7 @@ import { toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jsmConnectorMeta } from '@/connectors/jsm/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { htmlToPlainText, parseTagDate } from '@/connectors/utils' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' @@ -21,14 +21,25 @@ type JsmRequestStatus = (typeof VALID_REQUEST_STATUS)[number] /** * Allowed `requestOwnership` filter values for `GET /rest/servicedeskapi/request`. * - * This param scopes results to the OAuth user's relationship to each request. When - * omitted, the JSM API defaults to `OWNED_REQUESTS` — i.e. only requests the - * authenticated user reported. For a knowledge-base sync the user almost always - * wants every request in the service desk, so the connector defaults this to - * `ALL_REQUESTS` (which the JSM API treats as "owned + participated") rather than - * relying on the API's narrower default. + * This param scopes results to the OAuth user's relationship to each request — + * `GET /request` is always relative to the executing user, never a service-desk-wide + * dump. When omitted the API defaults to the combination `OWNED_REQUESTS`, + * `PARTICIPATED_REQUESTS`, and `ALL_ORGANIZATIONS`; `ALL_REQUESTS` is documented as + * "returns all customer requests" — the widest of the strategies — so it is the + * connector default. Atlassian marks it deprecated because the set it unions may + * change as strategies are added, not because it was narrowed, so it is kept until a + * single non-deprecated value covers the same breadth. + * + * `ORGANIZATION` and `APPROVER` are omitted: the first is only meaningful alongside + * an `organizationId` the connector does not collect, and the second is already + * inside what `ALL_REQUESTS` returns. */ -const VALID_REQUEST_OWNERSHIP = ['OWNED_REQUESTS', 'PARTICIPATED_REQUESTS', 'ALL_REQUESTS'] as const +const VALID_REQUEST_OWNERSHIP = [ + 'OWNED_REQUESTS', + 'PARTICIPATED_REQUESTS', + 'ALL_ORGANIZATIONS', + 'ALL_REQUESTS', +] as const type JsmRequestOwnership = (typeof VALID_REQUEST_OWNERSHIP)[number] /** @@ -54,6 +65,7 @@ interface JsmDate { interface JsmRequest { issueId?: string issueKey?: string + summary?: string requestTypeId?: string serviceDeskId?: string createdDate?: JsmDate @@ -143,13 +155,27 @@ function resolveOptions(sourceConfig: Record): { /** * Extracts a plain-text value for a given request field id (e.g. `summary`, - * `description`) from a request's `requestFieldValues`. The JSM API returns - * `value` either as a plain string (wiki markup) or, for some rich-text fields, - * as an ADF document — both are handled. + * `description`) from a request's `requestFieldValues`. + * + * `renderedValue.html` is preferred when the API supplies it: rich-text fields + * come back in `value` as raw Jira wiki markup (`I need a new *mouse*`) or as an + * ADF document, and the rendered HTML is the only form that preserves list, + * table, and link text without markup noise. Falls back to `value` as a string + * or via ADF extraction. */ function getFieldText(request: JsmRequest, fieldId: string): string { const field = request.requestFieldValues?.find((f) => f.fieldId === fieldId) if (!field) return '' + + const rendered = field.renderedValue + if (rendered && typeof rendered === 'object') { + const html = (rendered as { html?: unknown }).html + if (typeof html === 'string' && html.trim()) { + const text = htmlToPlainText(html).trim() + if (text) return text + } + } + const { value } = field if (typeof value === 'string') return value if (value && typeof value === 'object') { @@ -159,6 +185,17 @@ function getFieldText(request: JsmRequest, fieldId: string): string { return '' } +/** + * Resolves a request's summary. `CustomerRequestDTO.summary` is always present, + * whereas `requestFieldValues` omits fields hidden on the request-type form — so + * the top-level field is preferred and the field value is only a fallback. + */ +function getSummary(request: JsmRequest): string { + const top = typeof request.summary === 'string' ? request.summary.trim() : '' + if (top) return top + return getFieldText(request, 'summary').trim() +} + /** * Resolves the best available "change indicator" timestamp for a request. * @@ -188,7 +225,7 @@ function getChangeIndicator(request: JsmRequest): string { function requestToStub(request: JsmRequest, domain: string): ExternalDocument { const issueId = String(request.issueId ?? '') const issueKey = request.issueKey ?? issueId - const summary = getFieldText(request, 'summary') || 'Untitled' + const summary = getSummary(request) || 'Untitled' const status = request.currentStatus?.status const bareDomain = domain @@ -228,7 +265,7 @@ function requestToStub(request: JsmRequest, domain: string): ExternalDocument { function buildContent(request: JsmRequest, comments: JsmComment[]): string { const parts: string[] = [] - const summary = getFieldText(request, 'summary') + const summary = getSummary(request) if (summary) parts.push(summary) const description = getFieldText(request, 'description') @@ -268,10 +305,63 @@ async function resolveCloudId( return cloudId } +/** + * Resolves a configured service desk identifier to the numeric service desk id. + * + * `GET /servicedesk/{serviceDeskId}` accepts either the numeric id or a project + * key, but the `serviceDeskId` filter on `GET /request` is typed `integer` — a + * project key entered in advanced mode would validate and then silently match + * nothing. Resolve it once per sync and cache it on `syncContext`. + */ +async function resolveServiceDeskId( + baseUrl: string, + accessToken: string, + configuredId: string, + syncContext?: Record +): Promise { + const trimmed = configuredId.trim() + if (/^\d+$/.test(trimmed)) return trimmed + + const cached = syncContext?.serviceDeskNumericId as string | undefined + if (cached) return cached + + const response = await fetchWithRetry(`${baseUrl}/servicedesk/${encodeURIComponent(trimmed)}`, { + method: 'GET', + headers: getJsmHeaders(accessToken), + }) + + if (!response.ok) { + throw new Error(`Failed to resolve service desk "${trimmed}": ${response.status}`) + } + + const data = (await response.json()) as { id?: string } + const resolved = data.id ? String(data.id) : '' + if (!resolved) { + throw new Error(`Service desk "${trimmed}" returned no id`) + } + + if (syncContext) syncContext.serviceDeskNumericId = resolved + return resolved +} + +/** + * Upper bound on comments pulled into a single document. A pathological request + * thread must not fan out into unbounded pagination or an unbounded in-memory + * array; the cap is logged when it engages. + */ +const MAX_COMMENTS_PER_REQUEST = 500 + /** * Fetches comments for a request, following offset pagination until the API * signals `isLastPage`. When `publicOnly` is true the `public=true` filter is * applied so internal/agent-only comments are excluded. + * + * Throws on a failed page rather than returning what it has. The document's + * `contentHash` is metadata-only, so a thread cut short by a transient error + * would be indexed as complete and never re-fetched; throwing lets the sync + * engine record the document as failed and retry it on the next run. The + * {@link MAX_COMMENTS_PER_REQUEST} cut is deliberate and deterministic instead — + * retrying would land in exactly the same place — so it only warns. */ async function fetchComments( baseUrl: string, @@ -305,11 +395,7 @@ async function fetchComments( }) if (!response.ok) { - logger.warn('Failed to fetch JSM comments', { - issueIdOrKey, - status: response.status, - }) - break + throw new Error(`Failed to fetch JSM comments for ${issueIdOrKey}: ${response.status}`) } const data = (await response.json()) as JsmPage @@ -317,6 +403,15 @@ async function fetchComments( comments.push(...values) if (data.isLastPage || values.length === 0) break + + if (comments.length >= MAX_COMMENTS_PER_REQUEST) { + logger.warn('Truncating JSM comment thread at cap', { + issueIdOrKey, + cap: MAX_COMMENTS_PER_REQUEST, + }) + break + } + start += values.length } @@ -369,8 +464,15 @@ export const jsmConnector: ConnectorConfig = { return { documents: [], hasMore: false } } - const params = new URLSearchParams({ + const resolvedServiceDeskId = await resolveServiceDeskId( + baseUrl, + accessToken, serviceDeskId, + syncContext + ) + + const params = new URLSearchParams({ + serviceDeskId: resolvedServiceDeskId, requestStatus, start: String(start), limit: String(Math.min(PAGE_SIZE, remaining)), @@ -400,13 +502,7 @@ export const jsmConnector: ConnectorConfig = { } const data = (await response.json()) as JsmPage - let requests = data.values ?? [] - - let slicedSome = false - if (maxRequests > 0 && requests.length > remaining) { - slicedSome = true - requests = requests.slice(0, remaining) - } + const requests = data.values ?? [] const documents = requests.map((request) => requestToStub(request, domain)) @@ -419,12 +515,10 @@ export const jsmConnector: ConnectorConfig = { * When `maxRequests` truncates the listing before the source is exhausted, * flag the run as capped so the sync engine skips deletion reconciliation — * otherwise unseen requests beyond the cap would be deleted on every sync. - * `slicedSome` covers truncation on the final page: requests dropped from - * this page still exist even when `isLastPage` is true. (The requested - * `limit` never exceeds the remaining budget, so a slice should be - * impossible — this is defense in depth against the API over-returning.) + * Reaching the cap exactly as the source runs out (`isLastPage`) is not a + * truncation, so it must still reconcile. */ - if (((reachedCap && !data.isLastPage) || slicedSome) && syncContext) { + if (reachedCap && !data.isLastPage && syncContext) { syncContext.listingCapped = true } @@ -449,15 +543,28 @@ export const jsmConnector: ConnectorConfig = { const cloudId = await resolveCloudId(domain, accessToken, syncContext) const baseUrl = getJsmApiBaseUrl(cloudId) - const requestUrl = `${baseUrl}/request/${encodeURIComponent(externalId)}?expand=status` + /** + * No `expand` is requested: `summary`, `requestFieldValues`, `currentStatus`, + * `reporter`, and `_links` are all returned by default. `expand=status` + * would only add the full status-transition history, which is unused. + */ + const requestUrl = `${baseUrl}/request/${encodeURIComponent(externalId)}` const response = await fetchWithRetry(requestUrl, { method: 'GET', headers: getJsmHeaders(accessToken), }) + /** + * The JSM REST docs document 404 as "Returned if the customer request does not + * exist" and 403 as "Returned if the user does not have permission to complete + * this request" — for a sync credential both amount to absence, since the + * connector can never widen its own visibility. A 401 ("Returned if the user is + * not logged in") is an invalid/expired token: a whole-sync fault that must + * surface as a failed document rather than silently dropping every request. + */ if (!response.ok) { if (response.status === 404) return null - if (response.status === 401 || response.status === 403) { + if (response.status === 403) { logger.warn('Access denied fetching JSM request', { externalId, status: response.status }) return null } @@ -499,10 +606,20 @@ export const jsmConnector: ConnectorConfig = { } } + /** + * `requestTypeId` is typed `integer` on `GET /request`; a non-numeric value + * would be rejected mid-sync rather than at configuration time. + */ + const requestTypeId = + typeof sourceConfig.requestTypeId === 'string' ? sourceConfig.requestTypeId.trim() : '' + if (requestTypeId && !/^\d+$/.test(requestTypeId)) { + return { valid: false, error: 'Request type ID must be a number' } + } + try { const cloudId = await getJiraCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS) const baseUrl = getJsmApiBaseUrl(cloudId) - const url = `${baseUrl}/servicedesk/${encodeURIComponent(serviceDeskId)}` + const url = `${baseUrl}/servicedesk/${encodeURIComponent(serviceDeskId.trim())}` const response = await fetchWithRetry( url, diff --git a/apps/sim/connectors/jsm/meta.ts b/apps/sim/connectors/jsm/meta.ts index 66503e029a3..8b4f79003f8 100644 --- a/apps/sim/connectors/jsm/meta.ts +++ b/apps/sim/connectors/jsm/meta.ts @@ -25,10 +25,13 @@ export const jsmConnectorMeta: ConnectorMeta = { /** * Requests embed a `reporter` user object whose `displayName` is surfaced * in document content and the Reporter tag. Atlassian only populates - * embedded user data when the user-read scope is granted, so request it - * here. Present in the `jira` OAuth provider config as `read:jira-user`. + * embedded user data when a user-read scope is granted. The granular sets + * documented for `GET /request` and `GET /request/{id}/comment` both name + * `read:user:jira`; `read:jira-user` is its classic counterpart. Both are + * present in the `jira` OAuth provider config. */ 'read:jira-user', + 'read:user:jira', 'offline_access', ], }, @@ -58,7 +61,7 @@ export const jsmConnectorMeta: ConnectorMeta = { type: 'short-input', canonicalParamId: 'serviceDeskId', mode: 'advanced', - placeholder: 'e.g. 1, 2', + placeholder: 'e.g. 1, 2 (or a project key)', required: true, }, { @@ -98,11 +101,12 @@ export const jsmConnectorMeta: ConnectorMeta = { type: 'dropdown', required: false, description: - 'Which requests the connected account can see. "Owned + participated" is the broadest scope a customer token can sync.', + 'Which requests to sync, relative to the connected account. Jira Service Management only ever returns requests the connected account is related to, so "All requests" means every request that account created, participated in, or can see through its organizations — not the entire service desk.', options: [ - { label: 'Owned + participated', id: 'ALL_REQUESTS' }, + { label: 'All requests', id: 'ALL_REQUESTS' }, { label: 'Owned only', id: 'OWNED_REQUESTS' }, { label: 'Participated only', id: 'PARTICIPATED_REQUESTS' }, + { label: "All of the account's organizations", id: 'ALL_ORGANIZATIONS' }, ], }, { diff --git a/apps/sim/connectors/linear/linear.ts b/apps/sim/connectors/linear/linear.ts index 825a7189458..3a1639b311f 100644 --- a/apps/sim/connectors/linear/linear.ts +++ b/apps/sim/connectors/linear/linear.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { linearConnectorMeta } from '@/connectors/linear/meta' @@ -31,8 +33,45 @@ function markdownToPlainText(md: string): string { return text } +interface LinearGraphQLBody { + data?: Record | null + errors?: unknown[] +} + +/** Header carrying the UTC epoch-millisecond timestamp at which the request budget refills. */ +const RATE_LIMIT_RESET_HEADER = 'X-RateLimit-Requests-Reset' + +/** + * Largest reset wait honored. Linear's request budget refills on a one-hour + * leaky-bucket period, so the advertised reset instant is usually far too + * distant to wait for inside a sync. `backoffWithJitter` clamps the header + * value to this ceiling rather than sleeping until the real reset; if the + * budget has not recovered within the retry allowance the sync fails and the + * next scheduled run picks it up. + */ +const MAX_RATE_LIMIT_WAIT_MS = 30_000 + +/** + * Detects Linear's `RATELIMITED` extension code anywhere in a GraphQL error array. + */ +function isRateLimitedErrors(errors: unknown[] | undefined): boolean { + if (!Array.isArray(errors)) return false + return errors.some((entry) => { + if (typeof entry !== 'object' || entry === null) return false + const extensions = (entry as { extensions?: unknown }).extensions + if (typeof extensions !== 'object' || extensions === null) return false + return (extensions as { code?: unknown }).code === 'RATELIMITED' + }) +} + /** * Executes a GraphQL query against the Linear API. + * + * Linear signals rate limiting with an HTTP **400** carrying a `RATELIMITED` + * extension code rather than a 429, so `fetchWithRetry`'s status-based retry + * never fires for it. This wrapper detects the code on any status and paces its + * own retries off `X-RateLimit-Requests-Reset` (UTC epoch milliseconds), falling + * back to jittered exponential backoff when the header is absent. */ async function linearGraphQL( accessToken: string, @@ -40,32 +79,64 @@ async function linearGraphQL( variables?: Record, retryOptions?: RetryOptions ): Promise> { - const response = await fetchWithRetry( - LINEAR_API, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, + const maxRateLimitRetries = retryOptions?.maxRetries ?? 3 + + for (let attempt = 1; ; attempt++) { + const response = await fetchWithRetry( + LINEAR_API, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + body: JSON.stringify({ query, variables }), }, - body: JSON.stringify({ query, variables }), - }, - retryOptions - ) + retryOptions + ) - if (!response.ok) { - const errorText = await response.text() - logger.error('Linear GraphQL request failed', { status: response.status, error: errorText }) - throw new Error(`Linear API error: ${response.status}`) - } + const bodyText = await response.text() + let json: LinearGraphQLBody | undefined + try { + json = JSON.parse(bodyText) as LinearGraphQLBody + } catch { + json = undefined + } - const json = (await response.json()) as { data?: Record; errors?: unknown[] } - if (json.errors) { - logger.error('Linear GraphQL errors', { errors: json.errors }) - throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`) - } + if (isRateLimitedErrors(json?.errors) && attempt <= maxRateLimitRetries) { + const resetHeader = response.headers.get(RATE_LIMIT_RESET_HEADER) + const resetIn = resetHeader ? Number(resetHeader) - Date.now() : Number.NaN + const delayMs = backoffWithJitter(attempt, resetIn > 0 ? resetIn : null, { + maxMs: MAX_RATE_LIMIT_WAIT_MS, + }) + logger.warn('Linear rate limited; backing off', { attempt, delayMs }) + await sleep(delayMs) + continue + } - return json.data as Record + if (!response.ok) { + logger.error('Linear GraphQL request failed', { status: response.status, error: bodyText }) + throw new Error(`Linear API error: ${response.status}`) + } + + /** + * Linear reports validation, authorization, complexity, and rate-limit failures + * as HTTP 200 with a populated `errors` array and `data: null` (or a partially + * null `data`). Throwing here is deliberate: returning the partial payload would + * let `listDocuments` surface an empty node list, which the sync engine would + * read as "the source has no issues" and hard-delete every stored document. + */ + if (Array.isArray(json?.errors) && json.errors.length > 0) { + logger.error('Linear GraphQL errors', { errors: json.errors }) + throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`) + } + + if (!json?.data || typeof json.data !== 'object') { + throw new Error('Linear API returned no data') + } + + return json.data + } } /** @@ -119,85 +190,107 @@ const ISSUE_FIELDS = ` project { name } ` +/** + * Linear's `issue` query takes `id: String!`, not `ID!` — a mismatched variable + * definition fails GraphQL validation and returns HTTP 200 with `errors[]`. + */ const ISSUE_BY_ID_QUERY = ` - query GetIssue($id: ID!) { + query GetIssue($id: String!) { issue(id: $id) { ${ISSUE_FIELDS} } } ` +/** + * Filters are passed as a single `IssueFilter` variable rather than interpolated + * into the query text, so a static document covers every filter combination and + * no user-controlled value ever reaches the query string. + */ +const LIST_ISSUES_QUERY = ` + query ListIssues($filter: IssueFilter, $first: Int!, $after: String) { + issues(filter: $filter, first: $first, after: $after) { + nodes { + ${ISSUE_FIELDS} + } + pageInfo { + hasNextPage + endCursor + } + } + } +` + const TEAMS_QUERY = ` query { teams { nodes { id name key } } } ` /** - * Dynamically builds a GraphQL issues query with only the filter clauses - * that have values, preventing null comparators from being sent to Linear. + * Linear documents 50 as the connection default and publishes no maximum for + * `first`, so the default is used as-is. Each issue node also expands a nested + * `labels` connection (itself defaulting to 50), and Linear rejects any single + * query costing more than 10,000 complexity points — raising this would push + * toward that ceiling for no pagination benefit. */ -function buildIssuesQuery( +const MAX_PAGE_SIZE = 50 + +/** + * Builds the `IssueFilter` argument, omitting comparators that have no value so + * Linear never receives a null comparator. Returns undefined when unfiltered. + */ +function buildIssuesFilter( sourceConfig: Record, teamIds: string[], - projectIds: string[] -): { - query: string - variables: Record -} { - const stateFilter = (sourceConfig.stateFilter as string) || '' - - const varDefs: string[] = ['$first: Int!', '$after: String'] - const filterClauses: string[] = [] - const variables: Record = {} - - if (teamIds.length === 1) { - varDefs.push('$teamId: ID!') - filterClauses.push('team: { id: { eq: $teamId } }') - variables.teamId = teamIds[0] - } else if (teamIds.length > 1) { - varDefs.push('$teamIds: [ID!]!') - filterClauses.push('team: { id: { in: $teamIds } }') - variables.teamIds = teamIds - } + projectIds: string[], + lastSyncAt?: Date +): Record | undefined { + const filter: Record = {} - if (projectIds.length === 1) { - varDefs.push('$projectId: ID!') - filterClauses.push('project: { id: { eq: $projectId } }') - variables.projectId = projectIds[0] - } else if (projectIds.length > 1) { - varDefs.push('$projectIds: [ID!]!') - filterClauses.push('project: { id: { in: $projectIds } }') - variables.projectIds = projectIds - } + if (teamIds.length === 1) filter.team = { id: { eq: teamIds[0] } } + else if (teamIds.length > 1) filter.team = { id: { in: teamIds } } - if (stateFilter) { - const states = stateFilter - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - if (states.length > 0) { - varDefs.push('$stateFilter: [String!]!') - filterClauses.push('state: { name: { in: $stateFilter } }') - variables.stateFilter = states - } - } + if (projectIds.length === 1) filter.project = { id: { eq: projectIds[0] } } + else if (projectIds.length > 1) filter.project = { id: { in: projectIds } } - const filterArg = filterClauses.length > 0 ? `, filter: { ${filterClauses.join(', ')} }` : '' + const states = parseMultiValue(sourceConfig.stateFilter) + if (states.length > 0) filter.state = { name: { in: states } } - const query = ` - query ListIssues(${varDefs.join(', ')}) { - issues(first: $first, after: $after${filterArg}) { - nodes { - ${ISSUE_FIELDS} - } - pageInfo { - hasNextPage - endCursor - } - } - } - ` + if (lastSyncAt) filter.updatedAt = { gte: lastSyncAt.toISOString() } - return { query, variables } + return Object.keys(filter).length > 0 ? filter : undefined +} + +/** + * Projects a Linear issue node into an ExternalDocument. Shared by + * `listDocuments` and `getDocument` so `contentHash` and `metadata` are + * byte-identical regardless of which path produced the document. + */ +function issueToDocument(issue: Record): ExternalDocument { + const labelNodes = ((issue.labels as Record)?.nodes || []) as Record< + string, + unknown + >[] + const identifier = (issue.identifier as string) || '' + const title = (issue.title as string) || 'Untitled' + + return { + externalId: issue.id as string, + title: identifier ? `${identifier}: ${title}` : title, + content: buildIssueContent(issue), + mimeType: 'text/plain' as const, + sourceUrl: (issue.url as string) || undefined, + contentHash: `linear:${issue.id}:${issue.updatedAt}`, + metadata: { + identifier: issue.identifier, + state: (issue.state as Record)?.name, + priority: issue.priorityLabel, + assignee: (issue.assignee as Record)?.name, + labels: labelNodes.map((l) => l.name as string), + team: (issue.team as Record)?.name, + project: (issue.project as Record)?.name, + lastModified: issue.updatedAt, + }, + } } export const linearConnector: ConnectorConfig = { @@ -207,65 +300,68 @@ export const linearConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise => { - const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0 - const pageSize = maxIssues > 0 ? Math.min(maxIssues, 50) : 50 + const parsedMax = Number(sourceConfig.maxIssues) + const maxIssues = Number.isFinite(parsedMax) && parsedMax > 0 ? Math.floor(parsedMax) : 0 + + const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = + maxIssues > 0 ? Math.max(0, maxIssues - previouslyFetched) : Number.POSITIVE_INFINITY + const pageSize = Math.max(1, Math.min(MAX_PAGE_SIZE, remaining)) const teamIds = parseMultiValue(sourceConfig.teamId) const projectIds = parseMultiValue(sourceConfig.projectId) - - const { query, variables } = buildIssuesQuery(sourceConfig, teamIds, projectIds) - const allVars = { ...variables, first: pageSize, after: cursor || undefined } + const filter = buildIssuesFilter(sourceConfig, teamIds, projectIds, lastSyncAt) logger.info('Listing Linear issues', { cursor, pageSize, teamFilterCount: teamIds.length, projectFilterCount: projectIds.length, + incremental: Boolean(lastSyncAt), }) - const data = await linearGraphQL(accessToken, query, allVars) - const issuesConn = data.issues as Record - const nodes = (issuesConn.nodes || []) as Record[] - const pageInfo = issuesConn.pageInfo as Record - - const documents: ExternalDocument[] = nodes.map((issue) => { - const content = buildIssueContent(issue) - const contentHash = `linear:${issue.id}:${issue.updatedAt}` - - const labelNodes = ((issue.labels as Record)?.nodes || []) as Record< - string, - unknown - >[] - - return { - externalId: issue.id as string, - title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`, - content, - mimeType: 'text/plain' as const, - sourceUrl: (issue.url as string) || undefined, - contentHash, - metadata: { - identifier: issue.identifier, - state: (issue.state as Record)?.name, - priority: issue.priorityLabel, - assignee: (issue.assignee as Record)?.name, - labels: labelNodes.map((l) => l.name as string), - team: (issue.team as Record)?.name, - project: (issue.project as Record)?.name, - lastModified: issue.updatedAt, - }, - } + const data = await linearGraphQL(accessToken, LIST_ISSUES_QUERY, { + filter, + first: pageSize, + after: cursor || undefined, }) + /** + * `issues` is declared `IssueConnection!`, so a missing connection means a + * malformed response, not an empty source. Defaulting it to `{}` would + * present an empty listing, which the sync engine reads as "every stored + * issue was deleted" — so this throws instead. + */ + const issuesConn = data.issues as Record | undefined + if (!issuesConn || typeof issuesConn !== 'object') { + throw new Error('Linear API returned no issues connection') + } + const nodes = (issuesConn.nodes || []) as Record[] + const pageInfo = (issuesConn.pageInfo || {}) as Record + + const documents = nodes.map(issueToDocument) const hasNextPage = Boolean(pageInfo.hasNextPage) const endCursor = (pageInfo.endCursor as string) || undefined - const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length + const totalFetched = previouslyFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxIssues > 0 && totalFetched >= maxIssues + /** + * The `maxIssues` cap hides issues that still exist in Linear. The sync + * engine hard-deletes every stored document absent from a full listing, so + * a capped listing must be flagged. `first` is already clamped to the + * remaining budget, so the cap only hides issues when Linear reports a + * further page; genuine exhaustion is left unflagged so real deletions still + * reconcile. The team/project/state filters are intentional scope, not a cap. + */ + if (hitLimit && hasNextPage && syncContext) { + syncContext.listingCapped = true + } + return { documents, nextCursor: hasNextPage && !hitLimit ? endCursor : undefined, @@ -278,45 +374,19 @@ export const linearConnector: ConnectorConfig = { sourceConfig: Record, externalId: string ): Promise => { - try { - const data = await linearGraphQL(accessToken, ISSUE_BY_ID_QUERY, { id: externalId }) - const issue = data.issue as Record | null - - if (!issue) return null - - const content = buildIssueContent(issue) - const contentHash = `linear:${issue.id}:${issue.updatedAt}` - - const labelNodes = ((issue.labels as Record)?.nodes || []) as Record< - string, - unknown - >[] - - return { - externalId: issue.id as string, - title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`, - content, - mimeType: 'text/plain' as const, - sourceUrl: (issue.url as string) || undefined, - contentHash, - metadata: { - identifier: issue.identifier, - state: (issue.state as Record)?.name, - priority: issue.priorityLabel, - assignee: (issue.assignee as Record)?.name, - labels: labelNodes.map((l) => l.name as string), - team: (issue.team as Record)?.name, - project: (issue.project as Record)?.name, - lastModified: issue.updatedAt, - }, - } - } catch (error) { - logger.error('Failed to get Linear issue', { - externalId, - error: toError(error).message, - }) - return null - } + const data = await linearGraphQL(accessToken, ISSUE_BY_ID_QUERY, { id: externalId }) + const issue = data.issue as Record | null + + /** + * `issue` is declared `Issue!`, so Linear reports a missing issue as a + * GraphQL error rather than a null node — this guard is defence against a + * malformed body only. Transport, auth, and GraphQL failures propagate out + * of `linearGraphQL` so the sync engine records a visible failed document + * rather than silently dropping the issue. + */ + if (!issue) return null + + return issueToDocument(issue) }, validateConfig: async ( diff --git a/apps/sim/connectors/linear/meta.ts b/apps/sim/connectors/linear/meta.ts index 875122b3d60..33f9e4604ff 100644 --- a/apps/sim/connectors/linear/meta.ts +++ b/apps/sim/connectors/linear/meta.ts @@ -10,6 +10,14 @@ export const linearConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'linear', requiredScopes: ['read'] }, + /** + * `IssueFilter.updatedAt` is a `DateComparator`, so a sync can narrow to + * issues touched since the last run instead of walking the full dataset. + * Latent today: `knowledgeConnector.syncMode` defaults to `'full'` and + * nothing writes it, so `shouldRunIncrementalSync` never selects this path. + */ + supportsIncrementalSync: true, + configFields: [ { id: 'teamSelector', diff --git a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts index 80b321cdb16..aa28a9e2696 100644 --- a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts +++ b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts @@ -249,18 +249,29 @@ async function fetchWorkbookItem( return (await response.json()) as WorkbookItem } -/** Lists the workbook's worksheets in tab order. */ -async function fetchWorksheets(accessToken: string, basePath: string): Promise { +/** + * Lists the workbook's worksheets in tab order. + * + * `truncated` reports that the walk stopped while Graph was still offering more + * sheets — either because `@odata.nextLink` pointed off the Graph origin and was + * refused, or because the `MAX_WORKSHEETS` bound was reached. The caller must turn + * that into `listingCapped`, otherwise the sync engine reconciles the unseen sheets + * away as deletions. + */ +async function fetchWorksheets( + accessToken: string, + basePath: string +): Promise<{ worksheets: Worksheet[]; truncated: boolean }> { const worksheets: Worksheet[] = [] let url: string | undefined = `${basePath}/workbook/worksheets?$select=id,name,position,visibility&$orderby=position` + let truncated = false /** * Graph paginates collection responses, so a workbook with more sheets than fit * in one page must follow `@odata.nextLink`. Reading only the first page would - * drop the remainder from the listing without setting `listingCapped`, and the - * sync engine would then reconcile those documents away as deleted. The walk is - * bounded by `MAX_WORKSHEETS`, whose truncation the caller does flag. + * drop the remainder from the listing, and the sync engine would then reconcile + * those documents away as deleted. */ while (url && worksheets.length <= MAX_WORKSHEETS) { const response = await fetchWithRetry(url, { @@ -274,10 +285,31 @@ async function fetchWorksheets(accessToken: string, basePath: string): Promise (a.position ?? 0) - (b.position ?? 0)) + + return { worksheets, truncated } } /** @@ -352,6 +384,8 @@ async function fetchRangeValues( interface WorkbookSnapshot { workbook: WorkbookItem | null worksheets: Worksheet[] + /** True when the worksheet walk stopped before Graph ran out of sheets. */ + worksheetsTruncated: boolean } /** @@ -373,8 +407,9 @@ async function loadWorkbookSnapshot( const pending = (async (): Promise => { const workbook = await fetchWorkbookItem(accessToken, basePath) - if (!workbook) return { workbook: null, worksheets: [] } - return { workbook, worksheets: await fetchWorksheets(accessToken, basePath) } + if (!workbook) return { workbook: null, worksheets: [], worksheetsTruncated: false } + const { worksheets, truncated } = await fetchWorksheets(accessToken, basePath) + return { workbook, worksheets, worksheetsTruncated: truncated } })() if (syncContext) { @@ -463,7 +498,7 @@ export const microsoftExcelConnector: ConnectorConfig = { ): Promise => { const { spreadsheetId, basePath } = resolveBasePath(sourceConfig) - const { workbook, worksheets } = await loadWorkbookSnapshot( + const { workbook, worksheets, worksheetsTruncated } = await loadWorkbookSnapshot( accessToken, basePath, spreadsheetId, @@ -485,13 +520,23 @@ export const microsoftExcelConnector: ConnectorConfig = { const scoped = sheetFilter === 'first' ? worksheets.slice(0, 1) : worksheets const selected = scoped.slice(0, MAX_WORKSHEETS) - if (selected.length < scoped.length && syncContext) { - logger.warn('Worksheet listing truncated by the connector cap', { + + /** + * The listing is short of the workbook's real sheet set when the connector cap + * trims it, or when the worksheet walk itself stopped early. `sheetFilter: 'first'` + * is excluded on purpose — it is a deliberate scope choice, not a truncation, so + * the unselected sheets must still reconcile as deletions. + */ + const capped = + selected.length < scoped.length || (sheetFilter !== 'first' && worksheetsTruncated) + if (capped) { + logger.warn('Worksheet listing truncated; suppressing deletion reconciliation', { spreadsheetId, - total: scoped.length, + listed: selected.length, cap: MAX_WORKSHEETS, + worksheetsTruncated, }) - syncContext.listingCapped = true + if (syncContext) syncContext.listingCapped = true } logger.info('Listing Microsoft Excel worksheets', { @@ -594,11 +639,16 @@ export const microsoftExcelConnector: ConnectorConfig = { logger.info('Skipping oversized worksheet range', { externalId }) return markSkipped(stub, 'Worksheet exceeds the connector size limit and was not indexed') } - logger.warn('Failed to extract content from worksheet', { - externalId, - error: toError(error).message, - }) - return null + /** + * Everything reaching here is a transport or Graph failure that survived + * `fetchWithRetry`. Returning `null` reads to the sync engine as "no content", + * which for a newly added worksheet is silent — no counter moves and nothing is + * logged. Rethrowing surfaces it instead: the engine hydrates deferred documents + * under `Promise.allSettled`, so one bad worksheet never aborts the run, and a + * rejection increments `docsFailed`, logs the externalId, and marks it as an + * unverified refresh so a tombstoned sheet is not resurrected on a failed fetch. + */ + throw toError(error) } }, diff --git a/apps/sim/connectors/microsoft-teams/meta.ts b/apps/sim/connectors/microsoft-teams/meta.ts index 1d3f1fe2148..5cb79cea453 100644 --- a/apps/sim/connectors/microsoft-teams/meta.ts +++ b/apps/sim/connectors/microsoft-teams/meta.ts @@ -13,7 +13,12 @@ export const microsoftTeamsConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'microsoft-teams', - requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All'], + /** + * `Team.ReadBasic.All` backs the team selector's `GET /me/joinedTeams` call, + * `Channel.ReadBasic.All` backs `GET /teams/{id}/channels`, and + * `ChannelMessage.Read.All` backs the channel message and reply reads. + */ + requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All', 'Team.ReadBasic.All'], }, configFields: [ diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts index 5355918f867..f63ec5c0a49 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts @@ -15,9 +15,23 @@ import { const logger = createLogger('MicrosoftTeamsConnector') -const GRAPH_API_BASE = 'https://graph.microsoft.com/v1.0' +const GRAPH_API_ORIGIN = 'https://graph.microsoft.com' +const GRAPH_API_BASE = `${GRAPH_API_ORIGIN}/v1.0` + +/** + * Graph caps `$top` on channel messages at 50 per page (default 20). + * https://learn.microsoft.com/graph/api/channel-list-messages + */ const MESSAGES_PER_PAGE = 50 +/** + * Hard ceiling on `@odata.nextLink` hops drained per channel. Message paging is + * driven entirely by server-issued skip tokens, so without a cap a channel that + * keeps returning pages (e.g. one filled with system event messages that the + * user-message filter discards) would loop indefinitely inside a single sync. + */ +const MAX_MESSAGE_PAGES = 200 + interface TeamsMessage { id: string messageType: string @@ -39,6 +53,10 @@ interface TeamsMessage { content: string } subject?: string | null + /** Populated by `$expand=replies`; absent on the reply objects themselves. */ + replies?: TeamsMessage[] + /** Present when a message has more replies than the expand page size. */ + 'replies@odata.nextLink'?: string } interface TeamsChannel { @@ -57,6 +75,34 @@ interface TeamsChannelsResponse { value: TeamsChannel[] } +/** + * Resolves a relative Graph path or an absolute `@odata.nextLink` to a request + * URL, refusing any absolute URL that does not point at Microsoft Graph. The + * access token travels in the `Authorization` header, so following a + * server-supplied link to another origin would hand that token to a third + * party. Mirrors `assertGraphNextPageUrl` used by the Graph tool routes. + */ +function resolveGraphUrl(path: string): string { + if (!path.startsWith('https://')) return `${GRAPH_API_BASE}${path}` + + const url = new URL(path.trim()) + if (url.origin !== GRAPH_API_ORIGIN) { + throw new Error('Refusing to follow a non-Microsoft Graph @odata.nextLink') + } + return url.toString() +} + +/** Carries the HTTP status so callers can tell a deleted channel from a fault. */ +class GraphApiError extends Error { + constructor( + readonly status: number, + body: string + ) { + super(`Microsoft Graph API error: ${status} ${body}`.trim()) + this.name = 'GraphApiError' + } +} + /** * Calls the Microsoft Graph API with the given path and access token. */ @@ -65,7 +111,7 @@ async function graphApiGet( accessToken: string, retryOptions?: Parameters[2] ): Promise { - const url = path.startsWith('https://') ? path : `${GRAPH_API_BASE}${path}` + const url = resolveGraphUrl(path) const response = await fetchWithRetry( url, @@ -81,77 +127,148 @@ async function graphApiGet( if (!response.ok) { const errorBody = await response.text().catch(() => '') - throw new Error(`Microsoft Graph API error: ${response.status} ${errorBody}`) + throw new GraphApiError(response.status, errorBody) } return (await response.json()) as T } /** - * Fetches all messages from a channel, up to a maximum count, handling pagination. + * Resolves the configured message budget, falling back to the default for + * missing, non-numeric, or non-positive values. + * + * `validateConfig` rejects those inputs on save, but a config written before + * validation tightened (or edited out-of-band) would otherwise yield `NaN` + * here, which makes every budget comparison false and returns every channel + * with zero messages — dropping it from the listing entirely. + */ +function resolveMaxMessages(value: unknown): number { + if (value === undefined || value === null || value === '') return DEFAULT_MAX_MESSAGES + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : DEFAULT_MAX_MESSAGES +} + +/** Real user/app posts, excluding system event messages and tombstones. */ +function isUserMessage(message: TeamsMessage): boolean { + return message.messageType === 'message' && !message.deletedDateTime +} + +/** A root channel message together with the replies retained for it. */ +interface TeamsThread { + root: TeamsMessage + replies: TeamsMessage[] +} + +/** + * Fetches conversation threads from a channel, newest first, up to a total + * message budget shared by root messages and their replies. + * + * `GET /teams/{id}/channels/{id}/messages` returns root messages *without* + * replies, so `$expand=replies` is required to capture threaded conversation + * content. `$top` and `$expand` are the only OData parameters this endpoint + * supports. */ async function fetchChannelMessages( accessToken: string, teamId: string, channelId: string, maxMessages: number -): Promise<{ messages: TeamsMessage[]; lastActivityTs?: string }> { - const allMessages: TeamsMessage[] = [] - let nextLink: string | undefined +): Promise<{ threads: TeamsThread[]; messageCount: number; lastActivityTs?: string }> { + const threads: TeamsThread[] = [] let lastActivityTs: string | undefined + let remaining = maxMessages + let truncated = false + let pages = 0 - const initialPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/messages?$top=${Math.min(MESSAGES_PER_PAGE, maxMessages)}` + let currentUrl = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/messages?$top=${Math.min(MESSAGES_PER_PAGE, maxMessages)}&$expand=replies` - let currentUrl: string = initialPath - - while (allMessages.length < maxMessages) { + while (currentUrl && remaining > 0 && pages < MAX_MESSAGE_PAGES) { const data = await graphApiGet(currentUrl, accessToken) - const messages = data.value || [] + pages += 1 - if (messages.length === 0) break + for (const message of data.value || []) { + if (!isUserMessage(message)) continue + if (remaining <= 0) { + truncated = true + break + } - // Filter to actual user messages (skip system/event messages) - const userMessages = messages.filter( - (msg) => msg.messageType === 'message' && !msg.deletedDateTime - ) + /** Replies arrive newest-first, matching the root ordering. */ + const replies = (message.replies || []).filter(isUserMessage) + if (message['replies@odata.nextLink']) truncated = true + + remaining -= 1 + const keptReplies = replies.slice(0, remaining) + remaining -= keptReplies.length + if (keptReplies.length < replies.length) truncated = true + + if (!lastActivityTs) { + /** + * Graph sorts channel messages by the last modified date of the entire + * reply chain, so the first thread is the most recently active one — + * but the freshest timestamp in it may belong to a reply, not the root. + */ + const candidates = [message, ...replies].map( + (m) => m.lastModifiedDateTime || m.createdDateTime + ) + lastActivityTs = candidates.reduce((a, b) => (Date.parse(b) > Date.parse(a) ? b : a)) + } - // Messages are sorted by lastModifiedDateTime (per Graph docs), so the first - // user message on the first page reflects the most recent activity. - if (!lastActivityTs && userMessages.length > 0) { - const first = userMessages[0] - lastActivityTs = first.lastModifiedDateTime || first.createdDateTime + threads.push({ root: message, replies: keptReplies }) } - allMessages.push(...userMessages) + const nextLink = data['@odata.nextLink'] + currentUrl = nextLink ?? '' + if (currentUrl && remaining <= 0) truncated = true + } + + if (currentUrl && pages >= MAX_MESSAGE_PAGES) truncated = true - nextLink = data['@odata.nextLink'] - if (!nextLink) break - currentUrl = nextLink + if (truncated) { + logger.warn('Microsoft Teams channel content truncated; indexed a partial message history', { + teamId, + channelId, + maxMessages, + pages, + threads: threads.length, + }) } - return { messages: allMessages.slice(0, maxMessages), lastActivityTs } + const messageCount = threads.reduce((total, thread) => total + 1 + thread.replies.length, 0) + return { threads, messageCount, lastActivityTs } } -/** - * Converts fetched messages into a single document content string. - * Each line: "[ISO timestamp] username: message text" - */ -function formatMessages(messages: TeamsMessage[]): string { - const lines: string[] = [] +/** Renders one message as "[ISO timestamp] username: text", or '' when blank. */ +function formatMessage(message: TeamsMessage, prefix: string): string { + const bodyText = + message.body?.contentType === 'html' + ? htmlToPlainText(message.body.content) + : (message.body?.content ?? '') - // Process in reverse so oldest messages come first - const chronological = [...messages].reverse() + if (!bodyText.trim()) return '' - for (const msg of chronological) { - const bodyText = - msg.body.contentType === 'html' ? htmlToPlainText(msg.body.content) : msg.body.content + const userName = + message.from?.user?.displayName || message.from?.application?.displayName || 'unknown' - if (!bodyText.trim()) continue + return `${prefix}[${message.createdDateTime}] ${userName}: ${bodyText}` +} - const timestamp = msg.createdDateTime - const userName = msg.from?.user?.displayName || msg.from?.application?.displayName || 'unknown' +/** + * Converts fetched threads into a single document content string, oldest first, + * with each thread's replies indented beneath its root message. + */ +function formatMessages(threads: TeamsThread[]): string { + const lines: string[] = [] - lines.push(`[${timestamp}] ${userName}: ${bodyText}`) + // Process in reverse so oldest threads come first + for (const thread of [...threads].reverse()) { + const rootLine = formatMessage(thread.root, '') + if (rootLine) lines.push(rootLine) + + for (const reply of [...thread.replies].reverse()) { + const replyLine = formatMessage(reply, ' ') + if (replyLine) lines.push(replyLine) + } } return lines.join('\n') @@ -163,7 +280,8 @@ function formatMessages(messages: TeamsMessage[]): string { async function resolveChannel( accessToken: string, teamId: string, - channelInput: string + channelInput: string, + retryOptions?: Parameters[2] ): Promise { const trimmed = channelInput.trim() @@ -174,7 +292,7 @@ async function resolveChannel( let currentUrl: string = initialPath do { - const data = await graphApiGet(currentUrl, accessToken) + const data = await graphApiGet(currentUrl, accessToken, retryOptions) const channels = data.value || [] // Try matching by ID first, then by display name (case-insensitive) @@ -210,9 +328,7 @@ export const microsoftTeamsConnector: ConnectorConfig = { throw new Error('At least one channel is required') } - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) logger.info('Syncing Microsoft Teams channels', { teamId, @@ -228,14 +344,14 @@ export const microsoftTeamsConnector: ConnectorConfig = { throw new Error(`Channel not found: ${channelInput}`) } - const { messages, lastActivityTs } = await fetchChannelMessages( + const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( accessToken, teamId, channel.id, maxMessages ) - const content = formatMessages(messages) + const content = formatMessages(threads) if (!content.trim()) { logger.info(`No messages found in channel: ${channel.displayName}`) continue @@ -254,7 +370,7 @@ export const microsoftTeamsConnector: ConnectorConfig = { contentHash, metadata: { channelName: channel.displayName, - messageCount: messages.length, + messageCount, lastActivity: lastActivityTs || undefined, description: channel.description || undefined, }, @@ -278,23 +394,20 @@ export const microsoftTeamsConnector: ConnectorConfig = { return null } - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) try { - // Fetch channel info - const channelPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(externalId)}` + const channelPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(externalId)}?$select=id,displayName,description` const channel = await graphApiGet(channelPath, accessToken) - const { messages, lastActivityTs } = await fetchChannelMessages( + const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( accessToken, teamId, externalId, maxMessages ) - const content = formatMessages(messages) + const content = formatMessages(threads) if (!content.trim()) return null const contentHash = await computeContentHash(content) @@ -310,17 +423,25 @@ export const microsoftTeamsConnector: ConnectorConfig = { contentHash, metadata: { channelName: channel.displayName, - messageCount: messages.length, + messageCount, lastActivity: lastActivityTs || undefined, description: channel.description || undefined, }, } } catch (error) { + /** + * Only a channel that is genuinely gone resolves to `null`. Every other + * failure is rethrown so the sync engine records a visible failed document + * instead of dropping the channel from the run with no counter and no log. + */ + if (error instanceof GraphApiError && (error.status === 404 || error.status === 410)) { + return null + } logger.warn('Failed to get Microsoft Teams channel document', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, @@ -346,7 +467,12 @@ export const microsoftTeamsConnector: ConnectorConfig = { try { for (const channelInput of channelInputs) { - const channel = await resolveChannel(accessToken, teamId, channelInput) + const channel = await resolveChannel( + accessToken, + teamId, + channelInput, + VALIDATE_RETRY_OPTIONS + ) if (!channel) { return { valid: false, error: `Channel not found: ${channelInput}` } } diff --git a/apps/sim/connectors/mintlify/mintlify.ts b/apps/sim/connectors/mintlify/mintlify.ts index f4b9592abf9..4b3aa3fe5d8 100644 --- a/apps/sim/connectors/mintlify/mintlify.ts +++ b/apps/sim/connectors/mintlify/mintlify.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { validateExternalUrl } from '@/lib/core/security/input-validation' import { type SecureFetchRetryOptions, @@ -28,6 +28,50 @@ const PAGE_MAX_BYTES = 1024 * 1024 /** Child sitemaps followed from a ``, bounding a hostile or huge index. */ const MAX_CHILD_SITEMAPS = 20 +/** + * Bound on `` entries accumulated across a sitemap index. + * + * {@link INDEX_MAX_BYTES} caps each sitemap document individually, but a + * `` multiplies that by {@link MAX_CHILD_SITEMAPS}, so the merged + * location list is the largest allocation on the discovery path and the only one + * not bounded by a single document's size. Hitting this cap truncates the + * listing, so it reports `truncated`. + */ +const MAX_SITEMAP_LOCATIONS = 50_000 + +/** + * Character cap Mintlify applies to the `llms.txt` it generates: "Automatically + * generated `llms.txt` files cannot exceed 100,000 characters. If your + * documentation exceeds this limit, Mintlify truncates the file and appends a + * note listing the number of omitted pages." + * (https://mintlify.com/docs/ai/llmstxt) + * + * A truncated index is a partial listing the sync engine cannot distinguish from + * deletions, so a body at or near the cap must suppress deletion reconciliation. + * The exact wording of the appended note is not published, so it is deliberately + * not matched — the documented length is the only reliable signal. + */ +const LLMS_TXT_MAX_CHARS = 100_000 + +/** + * Margin below {@link LLMS_TXT_MAX_CHARS} that still counts as truncated. + * + * Mintlify publishes the cap but not where it cuts, so the delivered body is + * assumed to land near — not exactly at — 100,000 characters (a cut at an entry + * boundary, plus the appended note). The margin absorbs that unknown. Erring + * wide costs a complete-but-large index its deletion reconciliation, which a + * forced full sync can still override; erring narrow would hard-delete the pages + * a truncated index omitted, which nothing recovers. Hence the generous margin. + */ +const LLMS_TXT_TRUNCATION_MARGIN = 5_000 + +/** A discovered page set plus whether the source index itself was incomplete. */ +interface MintlifyDiscovery { + pages: MintlifyPageLink[] + /** The index was cut short (Mintlify's `llms.txt` cap or the sitemap location cap). */ + truncated: boolean +} + /** A page discovered from the site's index file. */ interface MintlifyPageLink { /** Site-absolute path without the `.md` extension, e.g. `/docs/quickstart`. */ @@ -270,7 +314,7 @@ async function discoverFromSitemap( site: MintlifySite, accessToken: string, retryOptions?: SecureFetchRetryOptions -): Promise { +): Promise { const root = await fetchSiteText( `${site.baseUrl}/sitemap.xml`, accessToken, @@ -278,10 +322,14 @@ async function discoverFromSitemap( INDEX_MAX_BYTES, retryOptions ) - if (!root) return [] + if (!root) return { pages: [], truncated: false } if (!SITEMAP_INDEX_PATTERN.test(root.body)) { - return parseSitemap(sitemapLocations(root.body), site) + const rootLocations = sitemapLocations(root.body) + return { + pages: parseSitemap(rootLocations.slice(0, MAX_SITEMAP_LOCATIONS), site), + truncated: rootLocations.length > MAX_SITEMAP_LOCATIONS, + } } const allChildUrls = sitemapLocations(root.body) @@ -297,7 +345,16 @@ async function discoverFromSitemap( logger.info('Following Mintlify sitemap index', { children: allChildUrls.length }) const locations: string[] = [] + let truncated = false for (const childUrl of allChildUrls) { + if (locations.length >= MAX_SITEMAP_LOCATIONS) { + truncated = true + logger.warn('Mintlify sitemap index exceeded the location cap', { + cap: MAX_SITEMAP_LOCATIONS, + }) + break + } + const child = await fetchSiteText( childUrl, accessToken, @@ -318,7 +375,12 @@ async function discoverFromSitemap( locations.push(...sitemapLocations(child.body)) } - return parseSitemap(locations, site) + if (locations.length > MAX_SITEMAP_LOCATIONS) { + truncated = true + locations.length = MAX_SITEMAP_LOCATIONS + } + + return { pages: parseSitemap(locations, site), truncated } } /** @@ -358,7 +420,7 @@ async function discoverPages( site: MintlifySite, accessToken: string, retryOptions?: SecureFetchRetryOptions -): Promise { +): Promise { /** * The origin-level index is only consulted for a site published at the host * root. On a sub-path site (`https://example.com/docs`) it describes the whole @@ -385,10 +447,21 @@ async function discoverPages( ) if (!result) continue const pages = withinBasePath(parseLlmsTxt(result.body, site), site) - if (pages.length > 0) return pages + if (pages.length === 0) continue + + const truncated = result.body.length >= LLMS_TXT_MAX_CHARS - LLMS_TXT_TRUNCATION_MARGIN + if (truncated) { + logger.warn('Mintlify llms.txt is at its generated character cap; listing may be partial', { + indexUrl, + chars: result.body.length, + cap: LLMS_TXT_MAX_CHARS, + }) + } + return { pages, truncated } } - return withinBasePath(await discoverFromSitemap(site, accessToken, retryOptions), site) + const sitemap = await discoverFromSitemap(site, accessToken, retryOptions) + return { pages: withinBasePath(sitemap.pages, site), truncated: sitemap.truncated } } /** Elements whose *text content* is markup/data, never prose. */ @@ -471,7 +544,7 @@ export const mintlifyConnector: ConnectorConfig = { let pages = syncContext?.pages as MintlifyPageLink[] | undefined if (!pages) { - const discovered = await discoverPages(site, accessToken) + const { pages: discovered, truncated } = await discoverPages(site, accessToken) /** * An empty discovery means the site's index files were unreachable or * unparseable, not that the docs are empty — `validateConfig` refuses a @@ -489,16 +562,19 @@ export const mintlifyConnector: ConnectorConfig = { ? discovered.filter((page) => isUnderPath(page.path, pathPrefix)) : discovered - if (filtered.length > maxPages && syncContext) { - /** - * The listing is truncated while more pages exist, so deletion - * reconciliation must be suppressed — otherwise every page past the cap - * would be hard-deleted from the knowledge base. - */ + /** + * The listing is incomplete — either `maxPages` cut it while more pages + * exist, or the source index itself was truncated (Mintlify's 100k-char + * `llms.txt` cap, or the sitemap location cap). Deletion reconciliation + * must be suppressed in both cases; otherwise every page missing from the + * partial listing is hard-deleted from the knowledge base. + */ + if ((truncated || filtered.length > maxPages) && syncContext) { syncContext.listingCapped = true logger.info('Mintlify page listing capped', { discovered: filtered.length, maxPages, + indexTruncated: truncated, }) } @@ -536,46 +612,40 @@ export const mintlifyConnector: ConnectorConfig = { const pages = syncContext?.pages as MintlifyPageLink[] | undefined const listed = pages?.find((page) => page.path === path) - try { - /** - * Mintlify serves every page as raw Markdown at `{page}.md`. A site that - * does not (a non-Mintlify host that merely publishes an `llms.txt`, or a - * page removed from the Markdown route) answers 404 there, so the rendered - * HTML page is the fallback and gets stripped to text. - */ - const result = - (await fetchSiteText( - `${site.origin}${path === '/' ? '/index' : path}.md`, - accessToken, - 'text/markdown', - PAGE_MAX_BYTES - )) ?? - (await fetchSiteText(`${site.origin}${path}`, accessToken, 'text/html', PAGE_MAX_BYTES)) - if (!result) return null + /** + * Mintlify serves every page as raw Markdown at `{page}.md`. A site that + * does not (a non-Mintlify host that merely publishes an `llms.txt`, or a + * page removed from the Markdown route) answers 404 there, so the rendered + * HTML page is the fallback and gets stripped to text. Both routes 404ing is + * the only *documented* absence; every transport, status, or size failure + * propagates out of `fetchSiteText` so the sync engine records a visible + * failed document rather than reading the page as deleted. + */ + const result = + (await fetchSiteText( + `${site.origin}${path === '/' ? '/index' : path}.md`, + accessToken, + 'text/markdown', + PAGE_MAX_BYTES + )) ?? (await fetchSiteText(`${site.origin}${path}`, accessToken, 'text/html', PAGE_MAX_BYTES)) + if (!result) return null - /** - * The `.md` route serves Markdown, which is already plain text. An HTML - * body — from the fallback above, or from a rewrite that ignores the - * extension — is stripped so raw markup is never indexed. - */ - const isHtml = - result.contentType.includes('html') || /^\s*<(!doctype\s+html|html\b)/i.test(result.body) - const content = isHtml ? htmlPageToPlainText(result.body) : result.body.trim() - if (!content) return null + /** + * The `.md` route serves Markdown, which is already plain text. An HTML + * body — from the fallback above, or from a rewrite that ignores the + * extension — is stripped so raw markup is never indexed. + */ + const isHtml = + result.contentType.includes('html') || /^\s*<(!doctype\s+html|html\b)/i.test(result.body) + const content = isHtml ? htmlPageToPlainText(result.body) : result.body.trim() + if (!content) return null - const stub = pageToStub(listed ?? { path, title: titleFromPath(path) }, site) - return { - ...stub, - content, - contentDeferred: false, - contentHash: `mintlify:${path}:${await computeContentHash(content)}`, - } - } catch (error) { - logger.warn('Failed to fetch Mintlify page', { - path, - error: toError(error).message, - }) - return null + const stub = pageToStub(listed ?? { path, title: titleFromPath(path) }, site) + return { + ...stub, + content, + contentDeferred: false, + contentHash: `mintlify:${path}:${await computeContentHash(content)}`, } }, @@ -599,7 +669,7 @@ export const mintlifyConnector: ConnectorConfig = { } try { - const pages = await discoverPages(site, accessToken, VALIDATE_RETRY_OPTIONS) + const { pages } = await discoverPages(site, accessToken, VALIDATE_RETRY_OPTIONS) if (pages.length === 0) { return { valid: false, diff --git a/apps/sim/connectors/monday/monday.test.ts b/apps/sim/connectors/monday/monday.test.ts new file mode 100644 index 00000000000..525523704b7 --- /dev/null +++ b/apps/sim/connectors/monday/monday.test.ts @@ -0,0 +1,273 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ MondayIcon: () => null })) + +import { mondayConnector } from '@/connectors/monday/monday' + +interface MondayReply { + status?: number + body?: unknown +} + +/** + * Queues monday GraphQL replies in order. monday has a single endpoint, so calls + * are matched positionally; the recorded request bodies are returned for + * assertions on the query text and variables. + */ +function mockMonday(replies: MondayReply[]) { + const requests: { query: string; variables: Record }[] = [] + let call = 0 + mockFetchWithRetry.mockImplementation(async (_url: string, options: RequestInit) => { + requests.push(JSON.parse(String(options.body))) + const reply = replies[call++] ?? { body: { data: {} } } + const status = reply.status ?? 200 + return { + ok: status >= 200 && status < 300, + status, + json: async () => reply.body, + text: async () => JSON.stringify(reply.body ?? {}), + } as unknown as Response + }) + return requests +} + +function item(id: string, overrides: Record = {}) { + return { + id, + name: `Item ${id}`, + state: 'active', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-02-01T00:00:00Z', + url: `https://example.monday.com/boards/1/pulses/${id}`, + board: { id: '1', name: 'Board One' }, + group: { id: 'g1', title: 'Group One' }, + creator: { name: 'Ada' }, + column_values: [], + updates: [], + ...overrides, + } +} + +function boardsPage(items: unknown[], cursor: string | null = null): MondayReply { + return { + body: { data: { boards: [{ id: '1', name: 'Board One', items_page: { cursor, items } }] } }, + } +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('monday listDocuments', () => { + it('passes board ids as GraphQL variables rather than interpolating them', async () => { + const requests = mockMonday([boardsPage([item('10')])]) + + await mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + + expect(requests[0].variables).toEqual({ ids: ['1'], limit: 100 }) + expect(requests[0].query).not.toContain('1234') + expect(requests[0].query).toContain('$ids') + }) + + it('leaves listingCapped unset when maxItems lands exactly on source exhaustion', async () => { + mockMonday([boardsPage([item('10'), item('11')], null)]) + + const syncContext: Record = {} + const result = await mondayConnector.listDocuments( + 'token', + { boardIds: '1', maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxItems stops a board that still has a cursor', async () => { + mockMonday([boardsPage([item('10'), item('11')], 'cursor-2')]) + + const syncContext: Record = {} + await mondayConnector.listDocuments( + 'token', + { boardIds: '1', maxItems: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when maxItems hides items on the same page', async () => { + mockMonday([boardsPage([item('10'), item('11'), item('12')], null)]) + + const syncContext: Record = {} + const result = await mondayConnector.listDocuments( + 'token', + { boardIds: '1', maxItems: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBe(true) + }) + + it('throws instead of returning an empty listing when monday returns errors on HTTP 200', async () => { + mockMonday([ + { + body: { + data: { + boards: [{ id: '1', name: 'Board One', items_page: { cursor: null, items: [] } }], + }, + errors: [ + { message: 'Board not found', extensions: { code: 'ResourceNotFoundException' } }, + ], + }, + }, + ]) + + await expect( + mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + ).rejects.toThrow('ResourceNotFoundException') + }) + + it('throws when monday returns no data and no errors', async () => { + mockMonday([{ body: { data: null } }]) + + await expect( + mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + ).rejects.toThrow('no data') + }) + + it('advances to the next board once a board is exhausted', async () => { + mockMonday([boardsPage([item('10')], null)]) + + const result = await mondayConnector.listDocuments('token', { boardIds: '1,2' }, undefined, {}) + + expect(result.hasMore).toBe(true) + expect(result.nextCursor).toBeDefined() + }) +}) + +describe('monday content extraction', () => { + it('falls back to display_value for columns that do not populate text', async () => { + mockMonday([ + boardsPage([ + item('10', { + column_values: [ + { + id: 'status', + type: 'status', + text: 'Done', + column: { id: 'status', title: 'Status' }, + }, + { + id: 'mirror', + type: 'mirror', + text: null, + display_value: 'Mirrored Value', + column: { id: 'mirror', title: 'Mirror' }, + }, + { + id: 'formula', + type: 'formula', + text: null, + display_value: '42', + column: { id: 'formula', title: 'Formula' }, + }, + ], + }), + ]), + ]) + + const result = await mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + + expect(result.documents[0].content).toContain('Status: Done') + expect(result.documents[0].content).toContain('Mirror: Mirrored Value') + expect(result.documents[0].content).toContain('Formula: 42') + }) + + it('selects display_value fragments for the column types that need them', async () => { + const requests = mockMonday([boardsPage([item('10')])]) + + await mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + + expect(requests[0].query).toContain('... on MirrorValue { display_value }') + expect(requests[0].query).toContain('... on BoardRelationValue { display_value }') + expect(requests[0].query).toContain('... on FormulaValue { display_value }') + }) + + it('produces an identical contentHash from listDocuments and getDocument', async () => { + mockMonday([boardsPage([item('10')]), { body: { data: { items: [item('10')] } } }]) + + const listed = await mondayConnector.listDocuments('token', { boardIds: '1' }, undefined, {}) + const fetched = await mondayConnector.getDocument('token', {}, '10') + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + expect(fetched?.contentHash).toBe('monday:10:2026-02-01T00:00:00Z') + }) +}) + +describe('monday getDocument', () => { + it('returns null when the item is not found', async () => { + mockMonday([{ body: { data: { items: [] } } }]) + + expect(await mondayConnector.getDocument('token', {}, '404')).toBeNull() + }) + + /** + * A swallowed error would return `null`, which the sync engine reads as an + * absent document rather than a failure — the item would silently vanish from + * the run with no counter and no error row. + */ + it('throws rather than returning null when the API fails', async () => { + mockMonday([{ status: 500, body: { error_message: 'Internal server error' } }]) + + await expect(mondayConnector.getDocument('token', {}, '10')).rejects.toThrow('500') + }) +}) + +describe('monday validateConfig', () => { + it('rejects a negative maxItems without calling the API', async () => { + const requests = mockMonday([]) + + const result = await mondayConnector.validateConfig('token', { maxItems: '-1' }) + + expect(result.valid).toBe(false) + expect(requests).toHaveLength(0) + }) + + it('surfaces the monday error code on an HTTP 200 failure', async () => { + mockMonday([ + { + body: { errors: [{ message: 'Not Authenticated', extensions: { code: 'Unauthorized' } }] }, + }, + ]) + + const result = await mondayConnector.validateConfig('token', {}) + + expect(result.valid).toBe(false) + expect(result.error).toContain('Unauthorized') + }) + + it('sends the raw token and a pinned API-Version header', async () => { + mockMonday([{ body: { data: { me: { id: '1' } } } }]) + + await mondayConnector.validateConfig('token', {}) + + const headers = mockFetchWithRetry.mock.calls[0][1].headers + expect(headers.Authorization).toBe('token') + expect(headers['API-Version']).toMatch(/^\d{4}-\d{2}$/) + }) +}) diff --git a/apps/sim/connectors/monday/monday.ts b/apps/sim/connectors/monday/monday.ts index c4340559ddc..87050cec2e1 100644 --- a/apps/sim/connectors/monday/monday.ts +++ b/apps/sim/connectors/monday/monday.ts @@ -1,38 +1,39 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { mondayConnectorMeta } from '@/connectors/monday/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' const logger = createLogger('MondayConnector') -/** - * monday.com GraphQL endpoint. All requests are POSTed here. - * @see https://developer.monday.com/api-reference/docs/basics - */ -const MONDAY_API_URL = 'https://api.monday.com/v2' - -/** - * Stable monday.com API version pinned via the `API-Version` header. monday.com - * keeps at least three quarterly versions live; `2024-10` was deprecated on - * 2026-02-15, so this is pinned to the current stable release. - * @see https://developer.monday.com/api-reference/docs/api-versioning - */ -const MONDAY_API_VERSION = '2026-04' - /** Max items requested per `items_page` / `next_items_page` call (monday.com max is 500). */ const ITEMS_PAGE_SIZE = 100 -/** Max boards requested per `boards` listing page (monday.com max is 500). */ +/** Boards requested per `boards` listing page (monday.com's default is 25, max 500). */ const BOARDS_PAGE_SIZE = 100 -/** Max updates fetched per item for content extraction. */ +/** + * Bound on the offset-paginated `boards` drain. `boards` exposes no cursor and no + * total count, so an unbounded loop is only terminated by the API returning a + * short page — this caps the walk at 5,000 boards instead of spinning forever. + */ +const MAX_BOARD_PAGES = 50 + +/** Max updates fetched per item for content extraction (monday.com's max is 100). */ const UPDATES_LIMIT = 50 interface MondayColumnValue { id: string text: string | null + /** + * Present only on the column types that dropped `text`. Selected via inline + * fragments in {@link ITEM_FIELDS}. + */ + display_value?: string | null column: { id: string; title: string } | null } @@ -96,22 +97,84 @@ function decodeCursor(cursor?: string): CursorState { } } +interface MondayGraphQLError { + message?: string + extensions?: { code?: string; status_code?: number; retry_in_seconds?: number } + retry_in_seconds?: number +} + +interface MondayGraphQLBody { + data?: T | null + errors?: MondayGraphQLError[] + error_message?: string + error_code?: string +} + /** - * monday.com uses the raw access token in the `Authorization` header — it is NOT - * prefixed with "Bearer". The `API-Version` header pins the schema version. - * @see https://developer.monday.com/api-reference/docs/authentication + * monday's throttling error codes, matched case-insensitively by substring. + * + * The errors page documents `Rate Limit Exceeded`, `COMPLEXITY_BUDGET_EXHAUSTED` + * and `maxConcurrencyExceeded` as HTTP 429 — which `fetchWithRetry` already + * retries off the `Retry-After` header — plus `API_TEMPORARILY_BLOCKED` on HTTP + * 200. The rate-limits page adds `ComplexityException`, `DAILY_LIMIT_EXCEEDED`, + * `IP_RATE_LIMIT_EXCEEDED`, `Concurrency limit exceeded` and `Minute limit rate + * exceeded`, none of which carry a documented status. + * + * The whole family is matched on an HTTP 200 body and retried here, since the + * codes without a documented 429 never reach `fetchWithRetry`'s retry path. + * @see https://developer.monday.com/api-reference/docs/errors + * @see https://developer.monday.com/api-reference/docs/rate-limits */ -function mondayHeaders(accessToken: string): Record { - return { - 'Content-Type': 'application/json', - Authorization: accessToken, - 'API-Version': MONDAY_API_VERSION, +const MONDAY_THROTTLE_CODE = + /complexity|rate.?limit|maxconcurrency|concurrency.?limit|daily.?limit|minute.?limit|temporarily_blocked/i + +/** Ceiling on an honored `retry_in_seconds`, so a large reset cannot stall a sync. */ +const MAX_THROTTLE_WAIT_MS = 60_000 + +/** + * Extracts the throttle back-off, in ms, if any GraphQL error in the body is a + * complexity/rate-limit failure. monday reports the wait as a `retry_in_seconds` + * body field rather than a `Retry-After` header, so `fetchWithRetry`'s + * header-driven pacing never sees it. + */ +function throttleRetryMs(errors: MondayGraphQLError[] | undefined): number | null { + if (!Array.isArray(errors)) return null + for (const error of errors) { + const code = error?.extensions?.code + const isThrottle = + (typeof code === 'string' && MONDAY_THROTTLE_CODE.test(code)) || + error?.extensions?.status_code === 429 + if (!isThrottle) continue + const seconds = error.retry_in_seconds ?? error.extensions?.retry_in_seconds + return Number.isFinite(seconds) && Number(seconds) > 0 + ? Math.min(Number(seconds) * 1000, MAX_THROTTLE_WAIT_MS) + : 0 } + return null +} + +/** Renders a GraphQL error array into a message that preserves `extensions.code`. */ +function formatGraphQLErrors(errors: MondayGraphQLError[]): string { + const parts = errors + .map((error) => { + const code = error?.extensions?.code + const message = error?.message + if (code && message) return `${code}: ${message}` + return code || message || '' + }) + .filter(Boolean) + return parts.join('; ') || 'Unknown GraphQL error' } /** - * Executes a GraphQL query against the monday.com API, surfacing GraphQL-level - * errors (which return HTTP 200 with an `errors` array) as thrown errors. + * Executes a GraphQL query against the monday.com API. + * + * monday returns `200 – OK` for application-level errors, and those responses may + * carry a **partially populated** `data` object alongside `errors`. Throwing on any + * `errors` entry is deliberate: returning the partial payload would let + * `listDocuments` surface a short or empty item list, which the sync engine would + * read as "the board has no items" and hard-delete every stored document. + * @see https://developer.monday.com/api-reference/docs/errors */ async function mondayGraphQL( accessToken: string, @@ -119,46 +182,74 @@ async function mondayGraphQL( variables: Record = {}, retryOptions?: Parameters[2] ): Promise { - const response = await fetchWithRetry( - MONDAY_API_URL, - { - method: 'POST', - headers: mondayHeaders(accessToken), - body: JSON.stringify({ query, variables }), - }, - retryOptions - ) - - if (!response.ok) { - const errorText = await response.text().catch(() => '') - throw new Error( - `monday.com API HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}` + const maxThrottleRetries = retryOptions?.maxRetries ?? 3 + + for (let attempt = 1; ; attempt++) { + const response = await fetchWithRetry( + MONDAY_API_URL, + { + method: 'POST', + headers: mondayHeaders(accessToken), + body: JSON.stringify({ query, variables }), + }, + retryOptions ) - } - const data = (await response.json()) as { - data?: T - errors?: { message?: string }[] - error_message?: string - } + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error( + `monday.com API HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}` + ) + } - if (data.errors && data.errors.length > 0) { - const message = data.errors - .map((e) => e.message) - .filter(Boolean) - .join('; ') - throw new Error(`monday.com API error: ${message || 'Unknown GraphQL error'}`) - } - if (data.error_message) { - throw new Error(`monday.com API error: ${data.error_message}`) - } + const body = (await response.json()) as MondayGraphQLBody + + const throttleMs = throttleRetryMs(body.errors) + if (throttleMs !== null && attempt <= maxThrottleRetries) { + const delayMs = backoffWithJitter(attempt, throttleMs || null, { + maxMs: MAX_THROTTLE_WAIT_MS, + }) + logger.warn('Monday.com throttled the request; backing off', { attempt, delayMs }) + await sleep(delayMs) + continue + } + + if (body.errors && body.errors.length > 0) { + throw new Error(`monday.com API error: ${formatGraphQLErrors(body.errors)}`) + } + if (body.error_message) { + const code = body.error_code ? `${body.error_code}: ` : '' + throw new Error(`monday.com API error: ${code}${body.error_message}`) + } + + /** + * No errors but no payload either. Returning `undefined` would let callers + * read an empty item list off `data.boards`, which reconciles as a deletion + * of everything the board owns. + */ + if (body.data === null || body.data === undefined) { + throw new Error('monday.com API returned no data') + } - return data.data as T + return body.data + } } /** * GraphQL selection set for an item, shared between listing and single-item * fetches so the resolved fields stay in sync. + * + * `ColumnValue.text` is documented as "not every column supports the text + * value": mirror, board_relation (connect boards), dependency, and formula + * columns expose their readable content on `display_value` instead. Mirror and + * dependency return `null` for `text`; formula returns an empty string. Either + * way `columnValueText` falls through, but selecting only `text` would silently + * drop those columns from the indexed document, so each is pulled in via an + * inline fragment. + * @see https://developer.monday.com/api-reference/reference/column-values-v2 + * @see https://developer.monday.com/api-reference/reference/mirror + * @see https://developer.monday.com/api-reference/reference/formula + * @see https://developer.monday.com/api-reference/reference/dependency */ const ITEM_FIELDS = ` id @@ -174,6 +265,10 @@ const ITEM_FIELDS = ` id text column { id title } + ... on MirrorValue { display_value } + ... on BoardRelationValue { display_value } + ... on DependencyValue { display_value } + ... on FormulaValue { display_value } } updates(limit: ${UPDATES_LIMIT}) { id @@ -254,6 +349,15 @@ function itemToDocument( } } +/** + * Resolves the readable text of a column value, preferring `text` and falling + * back to `display_value` for the column types that do not populate `text` + * (mirror, board_relation, dependency, formula). + */ +function columnValueText(cv: MondayColumnValue): string { + return cv.text?.trim() || cv.display_value?.trim() || '' +} + /** * Formats an item's column values and updates into a plain-text document. The * resolved board is passed in so listing (which has a board fallback) and @@ -269,14 +373,14 @@ function formatItemContent(item: MondayItem, board: { id: string; name: string } if (item.created_at) parts.push(`Created: ${item.created_at}`) if (item.updated_at) parts.push(`Updated: ${item.updated_at}`) - const columns = item.column_values.filter((cv) => cv.text?.trim()) - if (columns.length > 0) { - parts.push('') - parts.push('--- Fields ---') - for (const cv of columns) { - const title = cv.column?.title?.trim() || cv.id - parts.push(`${title}: ${cv.text}`) - } + const fieldLines: string[] = [] + for (const cv of item.column_values) { + const value = columnValueText(cv) + if (!value) continue + fieldLines.push(`${cv.column?.title?.trim() || cv.id}: ${value}`) + } + if (fieldLines.length > 0) { + parts.push('', '--- Fields ---', ...fieldLines) } const updates = item.updates.filter((u) => u.text_body?.trim()) @@ -299,7 +403,8 @@ function formatItemContent(item: MondayItem, board: { id: string; name: string } */ async function resolveBoardIds( accessToken: string, - sourceConfig: Record + sourceConfig: Record, + syncContext?: Record ): Promise<{ id: string; name: string | null }[]> { const configured = parseMultiValue(sourceConfig.boardIds) if (configured.length > 0) { @@ -308,7 +413,7 @@ async function resolveBoardIds( const boards: { id: string; name: string | null }[] = [] let page = 1 - for (;;) { + for (; page <= MAX_BOARD_PAGES; page++) { const data = await mondayGraphQL<{ boards: { id: string; name: string | null }[] | null }>( accessToken, `query ($limit: Int!, $page: Int!) { @@ -321,9 +426,23 @@ async function resolveBoardIds( ) const batch = data.boards ?? [] boards.push(...batch) - if (batch.length < BOARDS_PAGE_SIZE) break - page += 1 + /** + * `boards` is offset-paginated with no cursor or total count: a short page + * (or an empty one) is the only exhaustion signal. + */ + if (batch.length < BOARDS_PAGE_SIZE) return boards } + + /** + * The drain bound was reached with a full final page, so boards beyond it were + * never enumerated and their items are missing from the listing. Block deletion + * reconciliation so the sync engine does not hard-delete their stored documents. + */ + logger.warn('Monday.com board enumeration hit the page bound; listing is incomplete', { + boardCount: boards.length, + maxBoardPages: MAX_BOARD_PAGES, + }) + if (syncContext) syncContext.listingCapped = true return boards } @@ -341,7 +460,7 @@ export const mondayConnector: ConnectorConfig = { const boards = (syncContext?.boards as { id: string; name: string | null }[] | undefined) ?? - (await resolveBoardIds(accessToken, sourceConfig)) + (await resolveBoardIds(accessToken, sourceConfig, syncContext)) if (syncContext) syncContext.boards = boards if (state.boardIndex >= boards.length) { @@ -412,7 +531,19 @@ export const mondayConnector: ConnectorConfig = { const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxItems > 0 && totalFetched >= maxItems - if (hitLimit && syncContext) syncContext.listingCapped = true + + /** + * `listingCapped` blocks the sync engine's deletion reconciliation, so it must + * only be set when the cap actually hid documents that still exist. A cap that + * lands exactly on source exhaustion (nothing sliced off this page, no + * `items_page` cursor, no boards left) is a complete listing and must stay + * reconcilable — otherwise deleted items are never removed from the KB. + */ + const moreAvailable = + documents.length < allDocuments.length || + Boolean(nextItemsCursor) || + state.boardIndex + 1 < boards.length + if (hitLimit && moreAvailable && syncContext) syncContext.listingCapped = true let nextCursor: string | undefined let hasMore = false @@ -435,31 +566,26 @@ export const mondayConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null - - const data = await mondayGraphQL<{ items: MondayItem[] | null }>( - accessToken, - `query ($ids: [ID!]) { - items(ids: $ids) { ${ITEM_FIELDS} } - }`, - { ids: [externalId] } - ) + if (!externalId) return null - const item = data.items?.[0] - if (!item) return null + const data = await mondayGraphQL<{ items: MondayItem[] | null }>( + accessToken, + `query ($ids: [ID!]) { + items(ids: $ids) { ${ITEM_FIELDS} } + }`, + { ids: [externalId] } + ) - const doc = itemToDocument(item) - if (!doc.content.trim()) return null + /** + * monday answers a deleted or inaccessible id with an empty `items` array + * rather than an error, so this is the only absence. Every other failure + * propagates out of `mondayGraphQL` and is recorded by the sync engine as a + * visible failed document instead of being read as a deletion. + */ + const item = data.items?.[0] + if (!item) return null - return doc - } catch (error) { - logger.warn('Failed to get Monday.com item', { - externalId, - error: toError(error).message, - }) - return null - } + return itemToDocument(item) }, validateConfig: async ( diff --git a/apps/sim/connectors/notion/notion.ts b/apps/sim/connectors/notion/notion.ts index 3904d7ddef2..00ff9e1ac30 100644 --- a/apps/sim/connectors/notion/notion.ts +++ b/apps/sim/connectors/notion/notion.ts @@ -10,6 +10,59 @@ const logger = createLogger('NotionConnector') const NOTION_API_VERSION = '2022-06-28' const NOTION_BASE_URL = 'https://api.notion.com/v1' +/** + * Notion allows an average of ~3 requests/second per connection, so the one + * place this connector fans out stays at or below that. + */ +const NOTION_CONCURRENCY = 3 + +/** Maximum nesting depth walked by {@link fetchBlockTree}. */ +const MAX_BLOCK_DEPTH = 5 + +/** Upper bound on blocks pulled for a single page, to bound time and memory. */ +const MAX_BLOCKS_PER_PAGE = 2000 + +/** + * A Notion block with its recursively fetched children attached. + */ +interface NotionBlock extends Record { + children?: NotionBlock[] +} + +/** + * Per-page traversal state for {@link fetchBlockTree}. + * + * `remaining` is the block budget still available; `truncated` records that the + * walk stopped before the page was exhausted, so the cut is logged instead of + * silently shrinking the indexed content. + */ +interface BlockWalkState { + remaining: number + truncated: boolean +} + +/** + * Block types that own their own document and must not be inlined into the + * parent page's content — they are listed and synced separately. + */ +const NON_RECURSIVE_BLOCK_TYPES = new Set(['child_page', 'child_database']) + +/** + * Container blocks whose children are rendered at the parent's indent level + * rather than one level deeper. + */ +const TRANSPARENT_CONTAINER_TYPES = new Set(['table', 'column_list', 'column', 'synced_block']) + +/** + * Notion caps every paginated endpoint at 100 results. When a `maxPages` cap is + * configured, the final request asks only for what is still needed. + */ +function pageSizeFor(maxPages: number, syncContext?: Record): number { + if (maxPages <= 0) return 100 + const fetched = (syncContext?.totalDocsFetched as number) ?? 0 + return Math.max(1, Math.min(100, maxPages - fetched)) +} + /** * Extracts the title from a Notion page's properties. */ @@ -31,78 +84,136 @@ function richTextToPlain(richText: Record[]): string { } /** - * Extracts plain text content from Notion blocks. + * Renders a single block's own text, excluding its children. + * + * Covers the block types that carry no `rich_text` field (`table_row` uses + * `cells`, `child_page`/`child_database` use `title`, media blocks use + * `caption`), which would otherwise contribute nothing to the indexed content. */ -function blocksToPlainText(blocks: Record[]): string { - return blocks - .map((block) => { - const type = block.type as string - const blockData = block[type] as Record | undefined - if (!blockData) return '' - - if (type === 'code') { - const richText = blockData.rich_text as Record[] | undefined - const language = (blockData.language as string) || '' - const code = richText ? richTextToPlain(richText) : '' - return language ? `\`\`\`${language}\n${code}\n\`\`\`` : `\`\`\`\n${code}\n\`\`\`` - } +function renderBlockSelf(type: string, blockData: Record): string { + if (type === 'code') { + const richText = blockData.rich_text as Record[] | undefined + const language = (blockData.language as string) || '' + const code = richText ? richTextToPlain(richText) : '' + return language ? `\`\`\`${language}\n${code}\n\`\`\`` : `\`\`\`\n${code}\n\`\`\`` + } - if (type === 'equation') { - const expression = (blockData.expression as string) || '' - return expression ? `$$${expression}$$` : '' - } + if (type === 'equation') { + const expression = (blockData.expression as string) || '' + return expression ? `$$${expression}$$` : '' + } - const richText = blockData.rich_text as Record[] | undefined - if (!richText) return '' - - const text = richTextToPlain(richText) - - switch (type) { - case 'heading_1': - return `# ${text}` - case 'heading_2': - return `## ${text}` - case 'heading_3': - return `### ${text}` - case 'bulleted_list_item': - return `- ${text}` - case 'numbered_list_item': - return `1. ${text}` - case 'to_do': { - const checked = (blockData.checked as boolean) ? '[x]' : '[ ]' - return `${checked} ${text}` - } - case 'quote': - return `> ${text}` - case 'callout': - return text - case 'toggle': - return text - default: - return text - } - }) - .filter(Boolean) - .join('\n\n') + if (type === 'table_row') { + const cells = blockData.cells as Record[][] | undefined + if (!Array.isArray(cells)) return '' + const rendered = cells.map((cell) => (Array.isArray(cell) ? richTextToPlain(cell) : '')) + return rendered.some(Boolean) ? rendered.join(' | ') : '' + } + + if (type === 'child_page' || type === 'child_database') { + return (blockData.title as string) || '' + } + + if (type === 'divider') return '---' + + if (type === 'bookmark' || type === 'embed' || type === 'link_preview') { + const url = (blockData.url as string) || '' + const caption = blockData.caption as Record[] | undefined + const captionText = Array.isArray(caption) ? richTextToPlain(caption) : '' + return [captionText, url].filter(Boolean).join(' ') + } + + const richText = blockData.rich_text as Record[] | undefined + if (!richText) { + // Media/file blocks carry their only text in `caption`. + const caption = blockData.caption as Record[] | undefined + return Array.isArray(caption) ? richTextToPlain(caption) : '' + } + + const text = richTextToPlain(richText) + + switch (type) { + case 'heading_1': + return `# ${text}` + case 'heading_2': + return `## ${text}` + case 'heading_3': + return `### ${text}` + case 'bulleted_list_item': + return `- ${text}` + case 'numbered_list_item': + return `1. ${text}` + case 'to_do': { + const checked = (blockData.checked as boolean) ? '[x]' : '[ ]' + return `${checked} ${text}` + } + case 'quote': + return `> ${text}` + default: + return text + } +} + +/** + * Extracts plain text content from a Notion block tree, indenting nested + * children so structure survives into the indexed text. + */ +function blocksToPlainText(blocks: NotionBlock[], depth = 0): string { + const indent = ' '.repeat(depth) + const parts: string[] = [] + + for (const block of blocks) { + const type = block.type as string + const blockData = block[type] as Record | undefined + const self = blockData ? renderBlockSelf(type, blockData) : '' + + if (self) { + parts.push( + indent + ? self + .split('\n') + .map((line) => (line ? indent + line : line)) + .join('\n') + : self + ) + } + + const children = block.children + if (children?.length) { + const childDepth = TRANSPARENT_CONTAINER_TYPES.has(type) ? depth : depth + 1 + const nested = blocksToPlainText(children, childDepth) + if (nested) parts.push(nested) + } + } + + return parts.join('\n\n') } /** - * Fetches all block children for a page, handling pagination. + * Fetches one level of block children, handling pagination. + * + * Throws on a non-ok response rather than returning a partial level: the caller + * stores a metadata-based `contentHash`, so silently persisting truncated + * content would make the truncation permanent (the hash matches on every later + * sync and the page is never re-fetched). */ -async function fetchAllBlocks( +async function fetchBlockChildren( accessToken: string, - pageId: string -): Promise[]> { - const allBlocks: Record[] = [] + blockId: string, + state: BlockWalkState +): Promise { + const level: NotionBlock[] = [] let cursor: string | undefined let hasMore = true - while (hasMore) { - const params = new URLSearchParams({ page_size: '100' }) + while (hasMore && state.remaining > 0) { + const params = new URLSearchParams({ + page_size: String(Math.min(100, state.remaining)), + }) if (cursor) params.append('start_cursor', cursor) const response = await fetchWithRetry( - `${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`, + `${NOTION_BASE_URL}/blocks/${encodeURIComponent(blockId)}/children?${params.toString()}`, { method: 'GET', headers: { @@ -113,17 +224,79 @@ async function fetchAllBlocks( ) if (!response.ok) { - logger.warn(`Failed to fetch blocks for page ${pageId}`, { status: response.status }) - break + throw new Error(`Failed to fetch blocks for ${blockId}: ${response.status}`) } const data = await response.json() - allBlocks.push(...(data.results || [])) + const results = (data.results || []) as NotionBlock[] + level.push(...results) + state.remaining -= results.length cursor = data.next_cursor ?? undefined hasMore = data.has_more === true } - return allBlocks + if (hasMore) state.truncated = true + + return level +} + +/** + * Recursively fetches a page's block tree. + * + * `/v1/blocks/{id}/children` returns only the first level of children, so any + * block with `has_children: true` (toggles, callouts, columns, tables, nested + * lists) must be expanded with a further request or its content is lost. + * Recursion is bounded by {@link MAX_BLOCK_DEPTH} and {@link MAX_BLOCKS_PER_PAGE}. + * + * The walk is sequential. Notion allows an average of ~3 requests/second per + * connection, so parallelising the expansion only converts into 429s and + * backoff — and a per-level fan-out would compound to `n^depth` requests + * in flight, which is far past that limit. + */ +async function fetchBlockTree( + accessToken: string, + blockId: string, + depth: number, + state: BlockWalkState +): Promise { + if (depth > MAX_BLOCK_DEPTH || state.remaining <= 0) { + state.truncated = true + return [] + } + + const level = await fetchBlockChildren(accessToken, blockId, state) + + for (const block of level) { + if (block.has_children !== true) continue + if (NON_RECURSIVE_BLOCK_TYPES.has(block.type as string)) continue + if (state.remaining <= 0) { + state.truncated = true + break + } + block.children = await fetchBlockTree(accessToken, block.id as string, depth + 1, state) + } + + return level +} + +/** + * Fetches the complete block tree for a page, logging when the traversal is cut + * short by the depth or block bounds so truncation is never silent. + */ +async function fetchAllBlocks(accessToken: string, pageId: string): Promise { + const state: BlockWalkState = { remaining: MAX_BLOCKS_PER_PAGE, truncated: false } + const blocks = await fetchBlockTree(accessToken, pageId, 0, state) + + if (state.truncated) { + logger.warn('Notion page content truncated during block walk', { + pageId, + maxBlocks: MAX_BLOCKS_PER_PAGE, + maxDepth: MAX_BLOCK_DEPTH, + blocksFetched: MAX_BLOCKS_PER_PAGE - state.remaining, + }) + } + + return blocks } /** @@ -166,7 +339,15 @@ function pageToStub(page: Record): ExternalDocument { contentDeferred: true, mimeType: 'text/plain', sourceUrl: url, - contentHash: `notion:${pageId}:${lastEditedTime}`, + /** + * The `v2` namespace is a one-time invalidation. The hash is metadata-only, + * so a stored page whose `last_edited_time` has not moved is classified + * `unchanged` and never re-hydrated — meaning it would keep the truncated + * single-level block content indexed before recursive block fetching landed + * (tables in particular were indexed empty). Bumping the namespace forces + * one re-hydration per page, after which normal hash gating resumes. + */ + contentHash: `notion:v2:${pageId}:${lastEditedTime}`, metadata: { tags, lastModified: page.last_edited_time as string, @@ -209,13 +390,16 @@ export const notionConnector: ConnectorConfig = { externalId: string, _syncContext?: Record ): Promise => { - const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, - }, - }) + const response = await fetchWithRetry( + `${NOTION_BASE_URL}/pages/${encodeURIComponent(externalId)}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + }, + } + ) if (!response.ok) { if (response.status === 404) return null @@ -225,18 +409,17 @@ export const notionConnector: ConnectorConfig = { const page = await response.json() if (page.archived) return null - try { - const blocks = await fetchAllBlocks(accessToken, externalId) - const blockContent = blocksToPlainText(blocks) - const stub = pageToStub(page) - const content = blockContent.trim() || stub.title - return { ...stub, content, contentDeferred: false } - } catch (error) { - logger.warn(`Failed to fetch content for Notion page: ${externalId}`, { - error: toError(error).message, - }) - return null - } + /** + * A block-fetch failure propagates rather than degrading to `null`. The + * stored `contentHash` is metadata-based, so persisting a partial page + * would freeze the truncation in place; a thrown error is instead recorded + * by the sync engine as a document failure and retried on the next sync. + */ + const blocks = await fetchAllBlocks(accessToken, externalId) + const blockContent = blocksToPlainText(blocks) + const stub = pageToStub(page) + const content = blockContent.trim() || stub.title + return { ...stub, content, contentDeferred: false } }, validateConfig: async ( @@ -269,7 +452,7 @@ export const notionConnector: ConnectorConfig = { // Verify every database is accessible for (const databaseId of databaseIds) { const response = await fetchWithRetry( - `${NOTION_BASE_URL}/databases/${databaseId}`, + `${NOTION_BASE_URL}/databases/${encodeURIComponent(databaseId)}`, { method: 'GET', headers: { @@ -289,7 +472,7 @@ export const notionConnector: ConnectorConfig = { } else if (scope === 'page' && rootPageId) { // Verify page is accessible const response = await fetchWithRetry( - `${NOTION_BASE_URL}/pages/${rootPageId}`, + `${NOTION_BASE_URL}/pages/${encodeURIComponent(rootPageId)}`, { method: 'GET', headers: { @@ -357,7 +540,7 @@ async function listFromWorkspace( syncContext?: Record ): Promise { const body: Record = { - page_size: 100, + page_size: pageSizeFor(maxPages, syncContext), filter: { value: 'page', property: 'object' }, sort: { direction: 'descending', timestamp: 'last_edited_time' }, } @@ -467,7 +650,7 @@ async function listFromDatabases( while (databaseIndex < databaseIds.length) { const databaseId = databaseIds[databaseIndex] - const body: Record = { page_size: 100 } + const body: Record = { page_size: pageSizeFor(maxPages, syncContext) } if (startCursor) body.start_cursor = startCursor logger.info('Querying Notion database', { @@ -477,15 +660,18 @@ async function listFromDatabases( startCursor, }) - const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }) + const response = await fetchWithRetry( + `${NOTION_BASE_URL}/databases/${encodeURIComponent(databaseId)}/query`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + } + ) if (!response.ok) { const errorText = await response.text() @@ -550,13 +736,17 @@ async function listFromParentPage( cursor?: string, syncContext?: Record ): Promise { + // Always a full page of blocks: this endpoint pages over the root page's + // blocks, not over documents, and only the `child_page` ones become documents. + // Sizing the request by the remaining `maxPages` budget would shrink it to a + // handful of blocks per request and walk a long page in dozens of round-trips. const params = new URLSearchParams({ page_size: '100' }) if (cursor) params.append('start_cursor', cursor) logger.info('Listing child pages under root page', { rootPageId, cursor }) const response = await fetchWithRetry( - `${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`, + `${NOTION_BASE_URL}/blocks/${encodeURIComponent(rootPageId)}/children?${params.toString()}`, { method: 'GET', headers: { @@ -583,25 +773,33 @@ async function listFromParentPage( // Also include the root page itself on the first call (no cursor) const pageIdsToFetch = !cursor ? [rootPageId, ...childPageIds] : childPageIds - // Fetch page metadata (not content) in concurrent batches to build stubs - const CHILD_PAGE_CONCURRENCY = 5 - + // Fetch page metadata (not content) in concurrent batches to build stubs. + // A page dropped by a transient error still exists in Notion, so the listing + // is incomplete and deletion reconciliation must be suppressed — otherwise the + // sync engine hard-deletes the stored document. A 404 is genuine absence and + // does not cap the listing. const documents: ExternalDocument[] = [] - for (let i = 0; i < pageIdsToFetch.length; i += CHILD_PAGE_CONCURRENCY) { + let droppedByError = false + + for (let i = 0; i < pageIdsToFetch.length; i += NOTION_CONCURRENCY) { const cumulativeSoFar = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length if (maxPages > 0 && cumulativeSoFar >= maxPages) break - const batch = pageIdsToFetch.slice(i, i + CHILD_PAGE_CONCURRENCY) + const batch = pageIdsToFetch.slice(i, i + NOTION_CONCURRENCY) const results = await Promise.all( batch.map(async (pageId) => { try { - const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Notion-Version': NOTION_API_VERSION, - }, - }) + const pageResponse = await fetchWithRetry( + `${NOTION_BASE_URL}/pages/${encodeURIComponent(pageId)}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Notion-Version': NOTION_API_VERSION, + }, + } + ) if (!pageResponse.ok) { + if (pageResponse.status !== 404) droppedByError = true logger.warn(`Failed to fetch child page ${pageId}`, { status: pageResponse.status }) return null } @@ -609,6 +807,7 @@ async function listFromParentPage( if (page.archived) return null return pageToStub(page) } catch (error) { + droppedByError = true logger.warn(`Failed to process child page ${pageId}`, { error: toError(error).message, }) @@ -619,6 +818,8 @@ async function listFromParentPage( documents.push(...(results.filter(Boolean) as ExternalDocument[])) } + if (droppedByError && syncContext) syncContext.listingCapped = true + const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxPages > 0 && totalFetched >= maxPages diff --git a/apps/sim/connectors/obsidian/meta.ts b/apps/sim/connectors/obsidian/meta.ts index 19161273ffd..cb853e3b25f 100644 --- a/apps/sim/connectors/obsidian/meta.ts +++ b/apps/sim/connectors/obsidian/meta.ts @@ -21,7 +21,8 @@ export const obsidianConnectorMeta: ConnectorMeta = { type: 'short-input', placeholder: 'https://127.0.0.1:27124', required: true, - description: 'Base URL of your Obsidian Local REST API (default port: 27124 for HTTPS)', + description: + 'Base URL of your Obsidian Local REST API (default port: 27124 for HTTPS). The plugin ships a self-signed certificate, so the URL must be reachable over a trusted certificate — expose it through a reverse proxy with a valid certificate, or use the plugin HTTP port (27123) on a self-hosted Sim.', }, { id: 'folderPath', diff --git a/apps/sim/connectors/obsidian/obsidian.ts b/apps/sim/connectors/obsidian/obsidian.ts index 3620037e370..22f1167b8e3 100644 --- a/apps/sim/connectors/obsidian/obsidian.ts +++ b/apps/sim/connectors/obsidian/obsidian.ts @@ -1,27 +1,36 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { validateExternalUrl } from '@/lib/core/security/input-validation' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { obsidianConnectorMeta } from '@/connectors/obsidian/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { joinTagArray, parseTagDate } from '@/connectors/utils' +import { + CONNECTOR_MAX_FILE_BYTES, + joinTagArray, + markSkipped, + parseTagDate, + sizeLimitSkipReason, +} from '@/connectors/utils' const logger = createLogger('ObsidianConnector') const DOCS_PER_PAGE = 50 const DEFAULT_VAULT_URL = 'https://127.0.0.1:27124' +const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES interface NoteJson { content: string - frontmatter: Record + frontmatter?: Record path: string - stat: { + /** Optional in practice: a vault served through a proxy may omit it. */ + stat?: { ctime: number mtime: number size: number } - tags: string[] + tags?: string[] } /** @@ -36,7 +45,7 @@ interface NoteJson { * must be exposed through a public URL. */ function resolveVaultEndpoint(rawUrl: string | undefined): string { - let url = (rawUrl || DEFAULT_VAULT_URL).trim().replace(/\/+$/, '') + let url = (rawUrl || DEFAULT_VAULT_URL).trim() if (url && !url.startsWith('https://') && !url.startsWith('http://')) { url = `https://${url}` } @@ -44,7 +53,47 @@ function resolveVaultEndpoint(rawUrl: string | undefined): string { if (!validation.isValid) { throw new Error(validation.error || 'Invalid vault URL') } - return url + + /** + * Rebuilt from the parsed URL so a pasted value carrying a query string or + * fragment (`https://host:27124/?x=1`) cannot end up spliced into the middle + * of every endpoint path. A reverse-proxy base path is preserved. + */ + const parsed = new URL(url) + return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '') +} + +/** + * Normalizes a vault-relative path and rejects traversal segments. + * + * `encodeURIComponent` leaves `.` and `..` untouched, and `new URL()` resolves + * dot segments before the request is issued — so an unchecked `../..` in a + * config value or stored externalId would silently escape the plugin's + * `/vault/` prefix and address unrelated endpoints (`/commands/`, `/active/`). + */ +function normalizeVaultPath(rawPath: string, fieldName: string): string { + const segments = rawPath + .trim() + .split('/') + .filter((segment) => segment.length > 0) + + for (const segment of segments) { + if (segment === '.' || segment === '..') { + throw new Error(`${fieldName} must not contain path traversal segments`) + } + } + + return segments.join('/') +} + +/** Percent-encodes each segment of an already-normalized vault-relative path. */ +function encodeVaultPath(path: string): string { + return path.split('/').map(encodeURIComponent).join('/') +} + +/** Absolute URL of a note in the Local REST API vault namespace. */ +function noteUrl(baseUrl: string, filePath: string): string { + return `${baseUrl}/vault/${encodeVaultPath(filePath)}` } /** @@ -57,7 +106,7 @@ async function listDirectory( dirPath: string, retryOptions?: Parameters[2] ): Promise { - const encodedDir = dirPath ? dirPath.split('/').map(encodeURIComponent).join('/') : '' + const encodedDir = dirPath ? encodeVaultPath(dirPath) : '' const endpoint = encodedDir ? `${baseUrl}/vault/${encodedDir}/` : `${baseUrl}/vault/` const response = await secureFetchWithRetry( @@ -68,6 +117,7 @@ async function listDirectory( Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, + stripAuthOnRedirect: true, }, retryOptions ) @@ -82,15 +132,27 @@ async function listDirectory( const MAX_RECURSION_DEPTH = 20 +/** + * Tracks whether the recursive walk returned less than the vault actually holds. + * The sync engine hard-deletes every stored document absent from a listing, so a + * depth cut-off or a failed subdirectory read must surface as `listingCapped` + * rather than being read as evidence those notes were deleted. + */ +interface WalkState { + capped: boolean +} + async function listVaultFiles( baseUrl: string, accessToken: string, + state: WalkState, folderPath?: string, retryOptions?: Parameters[2], depth = 0 ): Promise { if (depth > MAX_RECURSION_DEPTH) { logger.warn('Max directory depth reached, skipping further recursion', { folderPath }) + state.capped = true return [] } @@ -112,9 +174,14 @@ async function listVaultFiles( for (const dir of subDirs) { try { - const nested = await listVaultFiles(baseUrl, accessToken, dir, retryOptions, depth + 1) + const nested = await listVaultFiles(baseUrl, accessToken, state, dir, retryOptions, depth + 1) mdFiles.push(...nested) } catch (error) { + /** + * A transient read failure drops every note under `dir` from this listing + * while they still exist in the vault — partial evidence, not deletion. + */ + state.capped = true logger.warn('Failed to list subdirectory', { dir, error: toError(error).message, @@ -131,20 +198,25 @@ async function listVaultFiles( async function fetchNote( baseUrl: string, accessToken: string, - filePath: string, - retryOptions?: Parameters[2] -): Promise { - const response = await secureFetchWithRetry( - `${baseUrl}/vault/${filePath.split('/').map(encodeURIComponent).join('/')}`, - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/vnd.olrapi.note+json', - }, + filePath: string +): Promise { + const response = await secureFetchWithRetry(noteUrl(baseUrl, filePath), { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/vnd.olrapi.note+json', }, - retryOptions - ) + stripAuthOnRedirect: true, + /** + * Bounds the buffered JSON envelope. The authoritative per-note gate is the + * `content` byte check in `getDocument`; this only stops a pathological + * body from being materialized in the sync worker's heap. + */ + maxResponseBytes: MAX_FILE_SIZE, + }) + + /** A note deleted between listing and hydration is an absence, not a failure. */ + if (response.status === 404) return null if (!response.ok) { throw new Error(`Obsidian API error fetching ${filePath}: ${response.status}`) @@ -161,6 +233,33 @@ function titleFromPath(filePath: string): string { return filename.replace(/\.md$/, '') } +/** + * Listing-time stub shared by `listDocuments` and `getDocument` so externalId, + * title, sourceUrl, and folder metadata are produced in exactly one place. + * + * `contentHash` is deliberately NOT part of the stub: the directory listing + * returns only `{ files: string[] }` (no `stat`, no `Last-Modified`/`ETag`, no + * `HEAD` route), so the stub hash cannot encode change-detection state and + * never matches the stored, mtime-bearing hash — every note is re-hydrated each + * sync. The engine's post-hydration compare + * (`priorByExternalId.get(id)?.contentHash === hydratedHash`) then skips the + * re-index, so this costs one GET per note but never re-embeds unchanged notes. + */ +function noteStub(baseUrl: string, filePath: string): ExternalDocument { + return { + externalId: filePath, + title: titleFromPath(filePath), + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: noteUrl(baseUrl, filePath), + contentHash: `obsidian:${filePath}`, + metadata: { + folder: filePath.includes('/') ? filePath.substring(0, filePath.lastIndexOf('/')) : '', + }, + } +} + export const obsidianConnector: ConnectorConfig = { ...obsidianConnectorMeta, @@ -171,40 +270,31 @@ export const obsidianConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const baseUrl = resolveVaultEndpoint(sourceConfig.vaultUrl as string) - const folderPath = (sourceConfig.folderPath as string) || '' + const folderPath = normalizeVaultPath((sourceConfig.folderPath as string) || '', 'folderPath') let allFiles = syncContext?.allFiles as string[] | undefined if (!allFiles) { logger.info('Listing all vault files', { baseUrl, folderPath }) - allFiles = await listVaultFiles(baseUrl, accessToken, folderPath || undefined) + const state: WalkState = { capped: false } + allFiles = await listVaultFiles(baseUrl, accessToken, state, folderPath || undefined) if (syncContext) { syncContext.allFiles = allFiles + /** + * Only ever set, never cleared: a walk that reached its depth limit or + * lost a subdirectory to a transient error returned fewer notes than the + * vault holds, and the engine must not read those absences as deletions. + * A clean, exhausted walk leaves the flag untouched so genuinely deleted + * notes still reconcile. + */ + if (state.capped) { + syncContext.listingCapped = true + } } } const offset = cursor ? Number(cursor) : 0 const pageFiles = allFiles.slice(offset, offset + DOCS_PER_PAGE) - /** - * The Obsidian Local REST API directory listing returns just - * `{ files: string[] }` — no `stat`/`mtime` and no `HEAD` support to read - * `Last-Modified`, so the stub cannot encode change-detection state. Every - * file is therefore re-hydrated via `getDocument` on every sync. The - * post-hydration hash compare in the sync engine - * (`existing.contentHash === hydratedHash`) prevents redundant DB writes - * when `mtime` is unchanged. - */ - const documents: ExternalDocument[] = pageFiles.map((filePath) => ({ - externalId: filePath, - title: titleFromPath(filePath), - content: '', - contentDeferred: true, - mimeType: 'text/plain' as const, - sourceUrl: `${baseUrl}/vault/${filePath.split('/').map(encodeURIComponent).join('/')}`, - contentHash: `obsidian:${filePath}`, - metadata: { - folder: filePath.includes('/') ? filePath.substring(0, filePath.lastIndexOf('/')) : '', - }, - })) + const documents: ExternalDocument[] = pageFiles.map((filePath) => noteStub(baseUrl, filePath)) const nextOffset = offset + pageFiles.length const hasMore = nextOffset < allFiles.length @@ -224,36 +314,55 @@ export const obsidianConnector: ConnectorConfig = { ): Promise => { const baseUrl = resolveVaultEndpoint(sourceConfig.vaultUrl as string) - try { - const note = await fetchNote(baseUrl, accessToken, externalId) - const content = note.content || '' + /** + * Throws rather than returning `null`: a traversal segment in a stored + * externalId is a rejection, not an absence, and `null` would drop the note + * from the run with no counter and no error row. + */ + const filePath = normalizeVaultPath(externalId, 'externalId') + const stub = noteStub(baseUrl, filePath) - return { - externalId, - title: titleFromPath(externalId), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `${baseUrl}/vault/${externalId.split('/').map(encodeURIComponent).join('/')}`, - contentHash: `obsidian:${externalId}:${note.stat?.mtime ?? ''}`, - metadata: { - tags: note.tags, - frontmatter: note.frontmatter, - createdAt: note.stat?.ctime ? new Date(note.stat.ctime).toISOString() : undefined, - modifiedAt: note.stat?.mtime ? new Date(note.stat.mtime).toISOString() : undefined, - size: note.stat?.size, - folder: externalId.includes('/') - ? externalId.substring(0, externalId.lastIndexOf('/')) - : '', - }, - } + let note: NoteJson | null + try { + note = await fetchNote(baseUrl, accessToken, filePath) } catch (error) { - logger.warn('Failed to get Obsidian note', { - externalId, - error: toError(error).message, - }) - return null + /** + * A note only proves oversized while its body streams, so that case becomes + * a visible skipped row. Every other failure is a transport or vault-API + * fault and is rethrown, letting the sync engine record a failed document + * instead of dropping the note from the run with no counter and no error. + */ + if (error instanceof PayloadSizeLimitError) { + return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) + } + logger.warn('Failed to get Obsidian note', { externalId, error: toError(error).message }) + throw toError(error) + } + if (!note) return null + + const content = note.content || '' + const hydrated: ExternalDocument = { + ...stub, + content, + contentDeferred: false, + contentHash: `obsidian:${filePath}:${note.stat?.mtime ?? ''}`, + metadata: { + ...stub.metadata, + tags: note.tags, + frontmatter: note.frontmatter, + createdAt: note.stat?.ctime ? new Date(note.stat.ctime).toISOString() : undefined, + modifiedAt: note.stat?.mtime ? new Date(note.stat.mtime).toISOString() : undefined, + size: note.stat?.size, + }, + } + + const contentBytes = Buffer.byteLength(content, 'utf8') + if (contentBytes > MAX_FILE_SIZE) { + logger.warn('Obsidian note exceeds size limit', { externalId, contentBytes }) + return markSkipped(hydrated, sizeLimitSkipReason(MAX_FILE_SIZE)) } + + return hydrated }, validateConfig: async ( @@ -278,6 +387,7 @@ export const obsidianConnector: ConnectorConfig = { { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, + stripAuthOnRedirect: true, }, VALIDATE_RETRY_OPTIONS ) @@ -299,17 +409,15 @@ export const obsidianConnector: ConnectorConfig = { } } - const folderPath = (sourceConfig.folderPath as string) || '' - if (folderPath.trim()) { - const entries = await listDirectory( - baseUrl, - accessToken, - folderPath.trim(), - VALIDATE_RETRY_OPTIONS - ) - if (entries.length === 0) { - logger.info('Folder path returned no entries', { folderPath }) - } + /** + * Proves the configured folder exists: the Local REST API answers 404 for + * an unknown directory, which `listDirectory` turns into a throw caught + * below. An empty-but-present folder is legitimate, so the entries + * themselves are not inspected. + */ + const folderPath = normalizeVaultPath((sourceConfig.folderPath as string) || '', 'folderPath') + if (folderPath) { + await listDirectory(baseUrl, accessToken, folderPath, VALIDATE_RETRY_OPTIONS) } return { valid: true } diff --git a/apps/sim/connectors/onedrive/onedrive.test.ts b/apps/sim/connectors/onedrive/onedrive.test.ts new file mode 100644 index 00000000000..a6236484ee8 --- /dev/null +++ b/apps/sim/connectors/onedrive/onedrive.test.ts @@ -0,0 +1,247 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ MicrosoftOneDriveIcon: () => null })) + +import { onedriveConnector } from '@/connectors/onedrive/onedrive' + +const GRAPH = 'https://graph.microsoft.com/v1.0' + +interface GraphRoute { + status?: number + body?: unknown +} + +function file(id: string, name: string, size = 10) { + return { + id, + name, + size, + file: { mimeType: 'text/plain' }, + webUrl: `https://example.com/${id}`, + lastModifiedDateTime: '2024-01-01T00:00:00Z', + } +} + +function folder(id: string, name: string) { + return { id, name, folder: { childCount: 1 } } +} + +/** Installs a URL-keyed fake Graph; unrouted URLs reply 404. */ +function mockGraph(routes: Record) { + const requested: string[] = [] + mockFetchWithRetry.mockImplementation(async (url: string) => { + requested.push(url) + const route = routes[url] ?? { status: 404 } + const status = route.status ?? 200 + return { + ok: status >= 200 && status < 300, + status, + json: async () => route.body, + text: async () => JSON.stringify(route.body ?? {}), + } as unknown as Response + }) + return requested +} + +const ROOT_URL = `${GRAPH}/me/drive/root/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference` +const childrenUrl = (id: string) => + `${GRAPH}/me/drive/items/${id}/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference` + +describe('onedrive listDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('walks nested folders within a single call', async () => { + const requested = mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('dir1', 'dir1')] } }, + [childrenUrl('dir1')]: { body: { value: [file('f2', 'b.md')] } }, + }) + + const syncContext: Record = {} + const result = await onedriveConnector.listDocuments('token', {}, undefined, syncContext) + + expect(requested).toHaveLength(2) + expect(result.documents.map((d) => d.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('follows @odata.nextLink pages of the same folder', async () => { + const nextLink = `${GRAPH}/me/drive/root/children?$skiptoken=abc` + mockGraph({ + [ROOT_URL]: { + body: { value: [file('f1', 'a.txt')], '@odata.nextLink': nextLink }, + }, + [nextLink]: { body: { value: [file('f2', 'b.txt')] } }, + }) + + const syncContext: Record = {} + const result = await onedriveConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((d) => d.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('leaves listingCapped unset when maxFiles lands exactly on source exhaustion', async () => { + mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), file('f2', 'b.txt')] } }, + }) + + const syncContext: Record = {} + const result = await onedriveConnector.listDocuments( + 'token', + { maxFiles: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxFiles hides items on the same page', async () => { + mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), file('f2', 'b.txt')] } }, + }) + + const syncContext: Record = {} + const result = await onedriveConnector.listDocuments( + 'token', + { maxFiles: '1' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when maxFiles lands on a page boundary with a nextLink left', async () => { + const nextLink = `${GRAPH}/me/drive/root/children?$skiptoken=abc` + mockGraph({ + [ROOT_URL]: { + body: { value: [file('f1', 'a.txt')], '@odata.nextLink': nextLink }, + }, + [nextLink]: { body: { value: [file('f2', 'b.txt')] } }, + }) + + const syncContext: Record = {} + const result = await onedriveConnector.listDocuments( + 'token', + { maxFiles: '1' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBe(true) + }) + + it('flags listingCapped when maxFiles stops traversal with folders pending', async () => { + mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('dir1', 'dir1')] } }, + [childrenUrl('dir1')]: { body: { value: [file('f2', 'b.txt')] } }, + }) + + const syncContext: Record = {} + await onedriveConnector.listDocuments('token', { maxFiles: '1' }, undefined, syncContext) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('resumes from the cursor when the per-call request budget is exhausted', async () => { + const routes: Record = { + [ROOT_URL]: { + body: { value: Array.from({ length: 30 }, (_, i) => folder(`dir${i}`, `dir${i}`)) }, + }, + } + for (let i = 0; i < 30; i++) { + routes[childrenUrl(`dir${i}`)] = { body: { value: [file(`f${i}`, `${i}.txt`)] } } + } + mockGraph(routes) + + const syncContext: Record = {} + const first = await onedriveConnector.listDocuments('token', {}, undefined, syncContext) + + expect(first.hasMore).toBe(true) + expect(first.nextCursor).toBeDefined() + + const second = await onedriveConnector.listDocuments('token', {}, first.nextCursor, syncContext) + + expect(first.documents.length + second.documents.length).toBe(30) + expect(second.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('encodes the configured folder path', async () => { + const url = `${GRAPH}/me/drive/root:/My%20Docs/Q1%20%26%20Q2:/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference` + const requested = mockGraph({ [url]: { body: { value: [] } } }) + + await onedriveConnector.listDocuments( + 'token', + { folderPath: '/My Docs/Q1 & Q2/' }, + undefined, + {} + ) + + expect(requested[0]).toBe(url) + }) +}) + +describe('onedrive getDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('returns null on 404', async () => { + mockGraph({}) + const doc = await onedriveConnector.getDocument!('token', {}, 'missing') + expect(doc).toBeNull() + }) + + it('produces the same contentHash as the listing stub', async () => { + const item = file('f1', 'a.txt') + mockGraph({ + [ROOT_URL]: { body: { value: [item] } }, + [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference`]: + { body: item }, + }) + + const listed = await onedriveConnector.listDocuments('token', {}, undefined, {}) + + mockFetchWithRetry.mockImplementation(async (url: string) => { + if (url.endsWith('/content')) { + return { + ok: true, + status: 200, + body: null, + arrayBuffer: async () => new TextEncoder().encode('hello').buffer, + } as unknown as Response + } + return { + ok: true, + status: 200, + json: async () => item, + text: async () => '', + } as unknown as Response + }) + + const fetched = await onedriveConnector.getDocument!('token', {}, 'f1') + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + expect(fetched?.contentDeferred).toBe(false) + expect(fetched?.content).toBe('hello') + }) +}) diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index 5cb5316a721..ef6a4f1366d 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -35,7 +35,37 @@ const SUPPORTED_EXTENSIONS = new Set([ const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES -const GRAPH_BASE_URL = 'https://graph.microsoft.com/v1.0' +const GRAPH_API_ORIGIN = 'https://graph.microsoft.com' +const GRAPH_BASE_URL = `${GRAPH_API_ORIGIN}/v1.0` + +/** + * The exact driveItem fields the stub is built from. Graph returns the full + * driveItem otherwise, which is an order of magnitude larger per item. + */ +const ITEM_SELECT = 'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference' + +/** + * Requested page size for a children collection, matching Graph's own default. + * + * No `$orderby` accompanies it: `/children` accepts `$orderby` on `name`, `size`, + * and `lastModifiedDateTime`, but "in OneDrive for Business and SharePoint Server + * 2016, the orderby query string only works with name and url" — so a + * `lastModifiedDateTime` sort is silently ignored on exactly the drives this + * connector is most often pointed at. The listing order is therefore whatever the + * drive returns, which matters only for *which* files a `maxFiles` cap keeps. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-list-children — 200-item default page size, `$orderby` support + * @see https://learn.microsoft.com/en-us/onedrive/developer/rest-api/concepts/optional-query-parameters — the name/url restriction + */ +const PAGE_SIZE = 200 + +/** + * Folder pages listed within a single `listDocuments` call. The sync engine caps + * a sync at a fixed number of `listDocuments` pages, and a depth-first walk needs + * at least one request per folder — draining several folders per call keeps a + * drive with thousands of folders from silently truncating its listing. + */ +const MAX_LIST_REQUESTS_PER_CALL = 25 interface OneDriveItem { id: string @@ -45,9 +75,7 @@ interface OneDriveItem { size?: number webUrl?: string lastModifiedDateTime?: string - createdDateTime?: string createdBy?: { user?: { displayName?: string } } - lastModifiedBy?: { user?: { displayName?: string } } parentReference?: { path?: string } } @@ -70,7 +98,7 @@ function isSupportedTextFile(name: string): boolean { * Downloads the raw content of a OneDrive file. */ async function downloadFileContent(accessToken: string, fileId: string): Promise { - const url = `${GRAPH_BASE_URL}/me/drive/items/${fileId}/content` + const url = `${GRAPH_BASE_URL}/me/drive/items/${encodeURIComponent(fileId)}/content` const response = await fetchWithRetry(url, { method: 'GET', @@ -131,17 +159,71 @@ function fileToStub(item: OneDriveItem): ExternalDocument { } /** - * Builds the list URL for the configured folder path or root. + * Normalizes a user-supplied folder path into percent-encoded path segments, + * or `undefined` when the drive root is targeted. */ -function buildListUrl(folderPath?: string): string { +function encodeFolderPath(folderPath?: string): string | undefined { const trimmed = folderPath?.trim() - if (trimmed) { - // Normalize path: strip leading/trailing slashes - const normalized = trimmed.replace(/^\/+|\/+$/g, '') - const encoded = normalized.split('/').map(encodeURIComponent).join('/') - return `${GRAPH_BASE_URL}/me/drive/root:/${encoded}:/children` + if (!trimmed) return undefined + const normalized = trimmed.replace(/^\/+|\/+$/g, '') + if (!normalized) return undefined + return normalized.split('/').map(encodeURIComponent).join('/') +} + +/** + * Builds the children-listing URL for a subfolder id, or for the configured + * root path when no folder id is supplied. + */ +function buildListUrl(folderPath: string | undefined, folderId: string | undefined): string { + const query = `?$top=${PAGE_SIZE}&$select=${ITEM_SELECT}` + if (folderId) { + return `${GRAPH_BASE_URL}/me/drive/items/${encodeURIComponent(folderId)}/children${query}` + } + const encoded = encodeFolderPath(folderPath) + return encoded + ? `${GRAPH_BASE_URL}/me/drive/root:/${encoded}:/children${query}` + : `${GRAPH_BASE_URL}/me/drive/root/children${query}` +} + +/** + * Asserts a paging URL points at Microsoft Graph before it is followed with the + * bearer token in the `Authorization` header. The `@odata.nextLink` this connector + * follows is persisted into the sync cursor, so it round-trips through storage + * rather than arriving straight off a TLS response — a tampered cursor must never + * be able to redirect the access token to a third-party host. Mirrors + * `assertGraphNextPageUrl` used by the Graph tool routes. + */ +function assertGraphNextLink(nextLink: string): string { + const url = new URL(nextLink.trim()) + if (url.origin !== GRAPH_API_ORIGIN) { + throw new Error('Refusing to follow a non-Microsoft Graph @odata.nextLink') + } + return url.toString() +} + +/** + * Depth-first traversal position carried across `listDocuments` calls. + */ +interface OneDriveTraversalState { + /** Absolute `@odata.nextLink` for the current folder's next page, if any. */ + nextLink?: string + /** Item id of the folder being listed; `undefined` means the configured root. */ + currentFolder?: string + /** Subfolder ids discovered but not yet listed. */ + folderStack: string[] +} + +function decodeCursor(cursor: string): OneDriveTraversalState { + try { + const parsed = JSON.parse(cursor) as Partial + return { + nextLink: typeof parsed.nextLink === 'string' ? parsed.nextLink : undefined, + currentFolder: typeof parsed.currentFolder === 'string' ? parsed.currentFolder : undefined, + folderStack: Array.isArray(parsed.folderStack) ? parsed.folderStack : [], + } + } catch { + return { folderStack: [] } } - return `${GRAPH_BASE_URL}/me/drive/root/children` } export const onedriveConnector: ConnectorConfig = { @@ -155,104 +237,117 @@ export const onedriveConnector: ConnectorConfig = { ): Promise => { const folderPath = sourceConfig.folderPath as string | undefined - /** - * Cursor state encodes the current page URL and a queue of pending folder IDs - * for recursive traversal. On initial call, we start from the configured path. - */ - let pageUrl: string - let folderQueue: string[] = [] - - if (cursor) { - try { - const parsed = JSON.parse(cursor) as { pageUrl?: string; folderQueue?: string[] } - pageUrl = parsed.pageUrl || buildListUrl(folderPath) - folderQueue = parsed.folderQueue || [] - } catch { - pageUrl = cursor - } - } else { - const baseUrl = buildListUrl(folderPath) - const separator = baseUrl.includes('?') ? '&' : '?' - pageUrl = `${baseUrl}${separator}$orderby=lastModifiedDateTime desc` - } + const parsedMaxFiles = Number(sourceConfig.maxFiles) + const maxFiles = Number.isFinite(parsedMaxFiles) && parsedMaxFiles > 0 ? parsedMaxFiles : 0 - logger.info('Listing OneDrive files', { - url: pageUrl, - cursor: cursor ? 'continuation' : 'initial', - }) + const state: OneDriveTraversalState = cursor ? decodeCursor(cursor) : { folderStack: [] } - const response = await fetchWithRetry(pageUrl, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + const documents: ExternalDocument[] = [] + let totalFetched = (syncContext?.totalDocsFetched as number) ?? 0 - if (!response.ok) { - const errorText = await response.text() - logger.error('Failed to list OneDrive files', { - status: response.status, - error: errorText, + /** Set when the walk finished — either the source ran out or `maxFiles` stopped it. */ + let done = false + /** Set only when `maxFiles` actually hid still-listable items. */ + let cappedWithItemsLeft = false + + for (let request = 0; request < MAX_LIST_REQUESTS_PER_CALL; request++) { + const pageUrl = state.nextLink + ? assertGraphNextLink(state.nextLink) + : buildListUrl(folderPath, state.currentFolder) + + logger.info('Listing OneDrive files', { + folderId: state.currentFolder ?? 'root', + pending: state.folderStack.length, + continuation: Boolean(state.nextLink), }) - throw new Error(`Failed to list OneDrive files: ${response.status}`) - } - const data = (await response.json()) as OneDriveListResponse - const items = data.value || [] + const response = await fetchWithRetry(pageUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) - // Collect subfolder IDs for recursive traversal - for (const item of items) { - if (item.folder) { - folderQueue.push(item.id) + if (!response.ok) { + const errorText = await response.text() + logger.error('Failed to list OneDrive files', { + status: response.status, + error: errorText, + }) + throw new Error(`Failed to list OneDrive files: ${response.status}`) } - } - // Keep oversized files and surface them as skipped (failed) documents instead - // of filtering them out silently. - const supportedFiles = items.filter((item) => item.file && isSupportedTextFile(item.name)) + const data = (await response.json()) as OneDriveListResponse + const items = data.value || [] - const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0 - const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const files: OneDriveItem[] = [] + for (const item of items) { + if (item.folder) { + state.folderStack.push(item.id) + } else if (item.file && isSupportedTextFile(item.name)) { + // Keep oversized files; they are surfaced as skipped (failed) docs below. + files.push(item) + } + } - const stubs = supportedFiles.map((item) => - stubOrSkipBySize(fileToStub(item), item.size, MAX_FILE_SIZE) - ) + const stubs = files.map((item) => + stubOrSkipBySize(fileToStub(item), item.size, MAX_FILE_SIZE) + ) + const take = takeIndexableWithinCap(stubs, isSkippedDocument, maxFiles, totalFetched) + documents.push(...take.documents) + totalFetched += take.indexableCount + + const nextLink = data['@odata.nextLink'] + + if (take.capReached) { + done = true + /** + * Only a cap that actually hid items makes the listing partial, and the + * cap can bite in three places: mid-page (`takeIndexableWithinCap` breaks + * out of the array early, so fewer stubs come back than went in), or at a + * page boundary with another page or another folder still pending. When + * the cap instead coincides with the last item of the last folder the + * source *is* fully listed, and flagging it capped would permanently + * block deletion reconciliation for a complete listing. + */ + cappedWithItemsLeft = + take.documents.length < stubs.length || Boolean(nextLink) || state.folderStack.length > 0 + break + } - const { documents, indexableCount, capReached } = takeIndexableWithinCap( - stubs, - isSkippedDocument, - maxFiles, - previouslyFetched - ) + if (nextLink) { + state.nextLink = nextLink + continue + } - const totalFetched = previouslyFetched + indexableCount - if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitLimit = capReached - if (hitLimit && syncContext) syncContext.listingCapped = true + if (state.folderStack.length > 0) { + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue + } - const nextLink = data['@odata.nextLink'] + done = true + break + } - // Determine next cursor: continue current page, or move to next queued folder - let nextCursor: string | undefined - let hasMore = false + if (syncContext) { + syncContext.totalDocsFetched = totalFetched + if (cappedWithItemsLeft) syncContext.listingCapped = true + } - if (!hitLimit) { - if (nextLink) { - nextCursor = JSON.stringify({ pageUrl: nextLink, folderQueue }) - hasMore = true - } else if (folderQueue.length > 0) { - const nextFolderId = folderQueue.shift()! - const nextUrl = `${GRAPH_BASE_URL}/me/drive/items/${nextFolderId}/children?$orderby=lastModifiedDateTime desc` - nextCursor = JSON.stringify({ pageUrl: nextUrl, folderQueue }) - hasMore = true - } + if (done) { + return { documents, hasMore: false } } + /** + * The per-call request budget ran out mid-walk. The engine keeps calling with + * this cursor, and flags the listing itself if it stops paging first. + */ return { documents, - nextCursor, - hasMore, + nextCursor: JSON.stringify(state), + hasMore: true, } }, @@ -261,7 +356,7 @@ export const onedriveConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - const url = `${GRAPH_BASE_URL}/me/drive/items/${externalId}` + const url = `${GRAPH_BASE_URL}/me/drive/items/${encodeURIComponent(externalId)}?$select=${ITEM_SELECT}` const response = await fetchWithRetry(url, { method: 'GET', @@ -291,10 +386,15 @@ export const onedriveConnector: ConnectorConfig = { logger.info('Skipping oversized OneDrive file', { fileId: item.id, name: item.name }) return markSkipped(fileToStub(item), sizeLimitSkipReason(error.limitBytes)) } + /** + * A transport or Graph failure that survived `fetchWithRetry`. Returning + * `null` would drop the file from the run with no `failed` row and no error + * log; rethrowing lets the sync engine record it per-document. + */ logger.warn(`Failed to fetch content for file: ${item.name} (${item.id})`, { error: toError(error).message, }) - return null + throw toError(error) } }, @@ -310,11 +410,10 @@ export const onedriveConnector: ConnectorConfig = { } try { - if (folderPath?.trim()) { + const encodedPath = encodeFolderPath(folderPath) + if (encodedPath) { // Verify the folder path exists and is accessible - const normalized = folderPath.trim().replace(/^\/+|\/+$/g, '') - const encoded = normalized.split('/').map(encodeURIComponent).join('/') - const url = `${GRAPH_BASE_URL}/me/drive/root:/${encoded}` + const url = `${GRAPH_BASE_URL}/me/drive/root:/${encodedPath}?$select=id,folder` const response = await fetchWithRetry( url, diff --git a/apps/sim/connectors/outlook/outlook.ts b/apps/sim/connectors/outlook/outlook.ts index c9123451511..25661fb0cda 100644 --- a/apps/sim/connectors/outlook/outlook.ts +++ b/apps/sim/connectors/outlook/outlook.ts @@ -56,6 +56,19 @@ const FULL_MESSAGE_FIELDS = [ */ const MAX_TOTAL_MESSAGES = 5000 +/** + * Hard cap Graph applies to a `$search` request on a message collection: + * "A `$search` request returns up to 1,000 results." + * + * Graph simply stops emitting `@odata.nextLink` at the cap, which is + * indistinguishable from mailbox exhaustion. A search-scoped listing that + * reaches it is therefore flagged as capped, otherwise deletion reconciliation + * would read every conversation past the 1,000th as deleted and hard-delete it. + * + * @see https://learn.microsoft.com/en-us/graph/search-query-parameter + */ +const GRAPH_SEARCH_RESULT_LIMIT = 1000 + interface OutlookEmailAddress { name?: string address?: string @@ -140,6 +153,26 @@ const MAX_FOLDER_RESOLUTION_REQUESTS = 25 */ const EXCLUDED_FOLDER_IDS_CONTEXT_KEY = '_outlookExcludedFolderIds' +/** + * Key under which `listDocuments` records the newest message date it saw per + * conversation, so the deferred `getDocument` hydration can reuse it verbatim. + * + * The listing filters messages (focused-inbox classification, `$search`, the + * date cutoff) that `getDocument` cannot reproduce in a `conversationId` + * lookup, so recomputing the date there yields a different `contentHash` than + * the stub. Since the sync engine stores the hydrated hash and compares the + * next listing's stub hash against it, that mismatch is permanent — the + * conversation would be re-fetched and re-indexed on every single sync. + */ +const CONVERSATION_LAST_DATE_CONTEXT_KEY = '_outlookConversationLastDates' + +/** + * Key under which the date cutoff is memoized for the sync run, so every page + * of a `$search` listing filters against the same instant rather than a + * per-page "now". + */ +const DATE_CUTOFF_CONTEXT_KEY = '_outlookDateCutoffMs' + const EMPTY_FOLDER_IDS: ReadonlySet = new Set() /** @@ -345,55 +378,78 @@ async function resolveExcludedFolderIds( } /** - * Builds the initial Graph API URL for listing messages. + * Builds the message-collection path for the configured folder. + * + * A folder that is not one of the well-known names is a Graph folder id coming + * from the folder selector, so it is URI-encoded before being spliced into the + * path. Well-known names are plain ASCII and encode to themselves. */ -function buildInitialUrl(sourceConfig: Record): string { +function buildMessagesBasePath(sourceConfig: Record): string { const folder = resolveFolder(sourceConfig) - const basePath = - folder === 'all' - ? `${GRAPH_API_BASE}/messages` - : `${GRAPH_API_BASE}/mailFolders/${WELL_KNOWN_FOLDERS[folder] || folder}/messages` + if (folder === 'all') return `${GRAPH_API_BASE}/messages` + const segment = WELL_KNOWN_FOLDERS[folder] ?? encodeURIComponent(folder) + return `${GRAPH_API_BASE}/mailFolders/${segment}/messages` +} +/** + * Returns the trimmed free-text search query, or `undefined` when unset. + */ +function resolveSearchQuery(sourceConfig: Record): string | undefined { + const query = sourceConfig.query + if (typeof query !== 'string') return undefined + return query.trim() || undefined +} + +/** + * Escapes a KQL clause for the `$search` parameter. + * + * Graph documents the clause as double-quote delimited: "The whole clause must + * be enclosed in double quotes. If it contains double quotes or backslash, + * escape it with a backslash." An unescaped quote in a user-supplied filter + * otherwise produces a malformed query that Graph rejects with a 400. + * + * @see https://learn.microsoft.com/en-us/graph/search-query-parameter + */ +function escapeSearchValue(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') +} + +/** + * Builds the initial Graph API URL for listing messages. + * + * Graph documents combining `$search` with `$filter` only for directory + * objects; for Outlook message collections it documents neither the combination + * nor which `$filter` properties survive one. Rather than depend on + * undocumented behavior, a search-scoped listing sends no `$filter` at all and + * applies the draft, focused-inbox, and date-cutoff predicates client-side. + */ +function buildInitialUrl(sourceConfig: Record): string { const params = new URLSearchParams({ $top: String(MESSAGES_PER_PAGE), $select: LIST_MESSAGE_FIELDS, }) - // Build $filter clauses - const filterParts: string[] = [] + const searchQuery = resolveSearchQuery(sourceConfig) - // Date range filter - const dateRange = (sourceConfig.dateRange as string) || 'all' - const dateIso = getDateRangeIso(dateRange) - if (dateIso) { - filterParts.push(`receivedDateTime ge ${dateIso}`) + if (searchQuery) { + params.set('$search', `"${escapeSearchValue(searchQuery)}"`) + return `${buildMessagesBasePath(sourceConfig)}?${params.toString()}` } - // When $search is active, Graph API restricts which $filter properties work. - // Apply isDraft and inferenceClassification filters client-side in that case. - const searchQuery = sourceConfig.query as string | undefined - const hasSearch = Boolean(searchQuery?.trim()) + const filterParts: string[] = ['isDraft eq false'] - if (!hasSearch) { - filterParts.push('isDraft eq false') + const dateIso = getDateRangeIso((sourceConfig.dateRange as string) || 'all') + if (dateIso) { + filterParts.push(`receivedDateTime ge ${dateIso}`) } - // Focused inbox filter — only apply server-side when no $search - const focusedOnly = sourceConfig.focusedOnly !== 'false' - if (focusedOnly && !hasSearch) { + if (sourceConfig.focusedOnly !== 'false') { filterParts.push("inferenceClassification eq 'focused'") } - if (filterParts.length > 0) { - params.set('$filter', filterParts.join(' and ')) - } - - // Free-text search (KQL syntax) - if (searchQuery?.trim()) { - params.set('$search', `"${searchQuery.trim()}"`) - } + params.set('$filter', filterParts.join(' and ')) - return `${basePath}?${params.toString()}` + return `${buildMessagesBasePath(sourceConfig)}?${params.toString()}` } /** @@ -427,6 +483,43 @@ function getDateRangeIso(dateRange: string): string | null { return date.toISOString() } +/** + * Resolves the date cutoff as epoch milliseconds for the client-side filtering + * that replaces the `receivedDateTime` `$filter` when `$search` is active. + * + * Memoized on the sync run so every page measures against the same instant — + * the server-side path is inherently stable because Graph replays the original + * `$filter` through `@odata.nextLink`. + */ +function resolveDateCutoffMs( + sourceConfig: Record, + syncContext?: Record +): number | null { + const cached = syncContext?.[DATE_CUTOFF_CONTEXT_KEY] + if (typeof cached === 'number') return cached + + const iso = getDateRangeIso((sourceConfig.dateRange as string) || 'all') + if (!iso) return null + + const cutoff = Date.parse(iso) + if (syncContext) syncContext[DATE_CUTOFF_CONTEXT_KEY] = cutoff + return cutoff +} + +/** + * Returns the messages ordered oldest-first. Graph does not guarantee an order + * without `$orderby`, and `$orderby` cannot be combined with this connector's + * `$filter` clauses without tripping Graph's `InefficientFilter` rule, so the + * ordering is established client-side. + */ +function sortByReceivedAscending(messages: OutlookMessage[]): OutlookMessage[] { + return [...messages].sort((a, b) => { + const dateA = a.receivedDateTime ? new Date(a.receivedDateTime).getTime() : 0 + const dateB = b.receivedDateTime ? new Date(b.receivedDateTime).getTime() : 0 + return dateA - dateB + }) +} + /** * Formats a recipient's display string. */ @@ -456,12 +549,7 @@ function formatConversation( ): { content: string; subject: string; metadata: Record } | null { if (messages.length === 0) return null - // Sort by receivedDateTime ascending (oldest first) - const sorted = [...messages].sort((a, b) => { - const dateA = a.receivedDateTime ? new Date(a.receivedDateTime).getTime() : 0 - const dateB = b.receivedDateTime ? new Date(b.receivedDateTime).getTime() : 0 - return dateA - dateB - }) + const sorted = sortByReceivedAscending(messages) const first = sorted[0] const last = sorted[sorted.length - 1] @@ -523,9 +611,10 @@ export const outlookConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { - const maxConversations = sourceConfig.maxConversations - ? Number(sourceConfig.maxConversations) - : DEFAULT_MAX_CONVERSATIONS + /** `validateConfig` rejects a non-positive value, so anything else here is drift. */ + const parsedMax = Number(sourceConfig.maxConversations) + const maxConversations = + Number.isFinite(parsedMax) && parsedMax > 0 ? parsedMax : DEFAULT_MAX_CONVERSATIONS // Initialize accumulator in syncContext if (syncContext && !syncContext._conversations) { @@ -565,9 +654,14 @@ export const outlookConnector: ConnectorConfig = { const data = await response.json() const messages = (data.value || []) as OutlookMessage[] - // Client-side filtering when $search is active (Graph API can't combine these with $search) + /** + * A search-scoped listing deliberately carries no `$filter` (see + * {@link buildInitialUrl}), so the draft, focused-inbox, and date-cutoff + * predicates are applied here instead. + */ const focusedOnly = sourceConfig.focusedOnly !== 'false' - const hasSearch = Boolean((sourceConfig.query as string)?.trim()) + const hasSearch = Boolean(resolveSearchQuery(sourceConfig)) + const dateCutoffMs = hasSearch ? resolveDateCutoffMs(sourceConfig, syncContext) : null const excludedFolderIds = isAllMailSync(sourceConfig) ? await resolveExcludedFolderIds(accessToken, syncContext) @@ -589,6 +683,11 @@ export const outlookConnector: ConnectorConfig = { continue } + if (dateCutoffMs !== null) { + const received = msg.receivedDateTime ? Date.parse(msg.receivedDateTime) : Number.NaN + if (Number.isFinite(received) && received < dateCutoffMs) continue + } + if (!msg.conversationId) continue const convId = msg.conversationId if (!conversations[convId]) { @@ -615,6 +714,15 @@ export const outlookConnector: ConnectorConfig = { * unvisited tail as deleted mail and hard-delete those documents. */ if (nextLink) syncContext.listingCapped = true + /** + * Graph stops paging a `$search` request at 1,000 results by simply + * omitting `@odata.nextLink`, which is indistinguishable from mailbox + * exhaustion. Treat reaching the cap as a truncated listing so the tail + * beyond it is not read as deleted mail and hard-deleted. + */ + if (hasSearch && newTotal >= GRAPH_SEARCH_RESULT_LIMIT) { + syncContext.listingCapped = true + } syncContext._fetchComplete = true } } @@ -650,6 +758,13 @@ export const outlookConnector: ConnectorConfig = { syncContext.listingCapped = true } + /** + * The hash date each stub was built from, handed to `getDocument` so the + * hydrated hash matches the stub exactly. See + * {@link CONVERSATION_LAST_DATE_CONTEXT_KEY}. + */ + const listedLastDates: Record = {} + const documents: ExternalDocument[] = [] for (const [convId, msgs] of limited) { if (msgs.length === 0) continue @@ -659,10 +774,14 @@ export const outlookConnector: ConnectorConfig = { return d > max ? d : max }, '') - const subject = msgs[0].subject || 'No Subject' - const firstWithLink = msgs.find((m) => m.webLink) + /** Oldest-first, matching how `formatConversation` picks the subject */ + const sorted = sortByReceivedAscending(msgs) + const subject = sorted[0].subject || 'No Subject' + const firstWithLink = sorted.find((m) => m.webLink) const sourceUrl = firstWithLink?.webLink || 'https://outlook.office.com/mail/inbox' + listedLastDates[convId] = lastDate + documents.push({ externalId: convId, title: subject, @@ -675,6 +794,10 @@ export const outlookConnector: ConnectorConfig = { }) } + if (syncContext) { + syncContext[CONVERSATION_LAST_DATE_CONTEXT_KEY] = listedLastDates + } + return { documents, hasMore: false } }, @@ -686,11 +809,7 @@ export const outlookConnector: ConnectorConfig = { ): Promise => { try { // Scope to the same folder as listDocuments so contentHash stays consistent - const folder = resolveFolder(sourceConfig) - const basePath = - folder === 'all' - ? `${GRAPH_API_BASE}/messages` - : `${GRAPH_API_BASE}/mailFolders/${WELL_KNOWN_FOLDERS[folder] || folder}/messages` + const basePath = buildMessagesBasePath(sourceConfig) const filterParts = [ `conversationId eq '${externalId.replace(/'/g, "''")}'`, @@ -723,11 +842,12 @@ export const outlookConnector: ConnectorConfig = { const allMessages = (data.value || []) as OutlookMessage[] /** - * Mirrors the listing's exclusion so `contentHash` is computed over the - * same message set on both sides. Without it, deleting the newest message - * of a conversation would leave the listing hash (recomputed from the - * surviving messages) permanently disagreeing with the hash returned - * here, re-fetching the conversation on every sync forever. + * Mirrors the listing's exclusion so the indexed content — and the + * fallback `contentHash` computed below when no listing date is + * available — covers the same message set on both sides. Without it, + * deleting the newest message of a conversation would leave the listing + * hash permanently disagreeing with the hash returned here, re-fetching + * the conversation on every sync forever. */ const excludedFolderIds = isAllMailSync(sourceConfig) ? await resolveExcludedFolderIds(accessToken, syncContext) @@ -739,12 +859,28 @@ export const outlookConnector: ConnectorConfig = { const result = formatConversation(externalId, messages) if (!result) return null - const lastDate = messages.reduce((max, m) => { - const d = m.receivedDateTime || '' - return d > max ? d : max - }, '') - - const firstWithLink = messages.find((m) => m.webLink) + /** + * Prefer the date the listing hashed this conversation with. The listing + * narrows messages by focused-inbox classification, `$search`, and the + * date cutoff — none of which can be replayed in a `conversationId` + * lookup — so recomputing here would produce a hash that permanently + * disagrees with the stub and re-index the conversation every sync. + */ + const listedLastDates = syncContext?.[CONVERSATION_LAST_DATE_CONTEXT_KEY] + const listedLastDate = + listedLastDates && typeof listedLastDates === 'object' + ? (listedLastDates as Record)[externalId] + : undefined + + const lastDate = + typeof listedLastDate === 'string' + ? listedLastDate + : messages.reduce((max, m) => { + const d = m.receivedDateTime || '' + return d > max ? d : max + }, '') + + const firstWithLink = sortByReceivedAscending(messages).find((m) => m.webLink) return { externalId, @@ -757,11 +893,17 @@ export const outlookConnector: ConnectorConfig = { metadata: result.metadata, } } catch (error) { + /** + * A transport or Graph failure that survived `fetchWithRetry`. Returning + * `null` would drop the conversation from the run with no `failed` row and + * no error log; rethrowing lets the sync engine record it per-document. + * Genuine absence is already handled above (404, or no matching messages). + */ logger.warn('Failed to get Outlook conversation', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, @@ -781,10 +923,7 @@ export const outlookConnector: ConnectorConfig = { try { // Verify Graph API access const folder = resolveFolder(sourceConfig) - const testUrl = - folder === 'all' - ? `${GRAPH_API_BASE}/messages?$top=1&$select=id` - : `${GRAPH_API_BASE}/mailFolders/${WELL_KNOWN_FOLDERS[folder] || folder}/messages?$top=1&$select=id` + const testUrl = `${buildMessagesBasePath(sourceConfig)}?$top=1&$select=id` const response = await fetchWithRetry( testUrl, @@ -806,14 +945,14 @@ export const outlookConnector: ConnectorConfig = { } // If a search query is specified, verify it's valid with a dry run - const searchQuery = sourceConfig.query as string | undefined - if (searchQuery?.trim()) { + const searchQuery = resolveSearchQuery(sourceConfig) + if (searchQuery) { const searchParams = new URLSearchParams({ - $search: `"${searchQuery.trim()}"`, + $search: `"${escapeSearchValue(searchQuery)}"`, $top: '1', $select: 'id', }) - const searchUrl = `${GRAPH_API_BASE}/messages?${searchParams.toString()}` + const searchUrl = `${buildMessagesBasePath(sourceConfig)}?${searchParams.toString()}` const searchResponse = await fetchWithRetry( searchUrl, { diff --git a/apps/sim/connectors/pagerduty/meta.ts b/apps/sim/connectors/pagerduty/meta.ts index 78312e5d37e..8978c69d847 100644 --- a/apps/sim/connectors/pagerduty/meta.ts +++ b/apps/sim/connectors/pagerduty/meta.ts @@ -15,13 +15,22 @@ export const pagerdutyConnectorMeta: ConnectorMeta = { }, /** - * Deliberately absent. PagerDuty's `since`/`until` filter incident *creation* - * time, and the REST API exposes no modified-since filter, so an incremental - * listing would never surface a status change, a new note, or a resolution on - * an incident created before the window — an incident synced while triggered - * would stay triggered forever. Every sync therefore lists the full history, - * gated by `contentHash` so unchanged incidents are never re-hydrated. + * Re-fetch every incident on an explicit full resync. + * + * The document's `contentHash` is keyed on the incident's `updated_at`, but its + * content also folds in notes and log entries — child resources PagerDuty does + * not document as bumping the parent incident's `updated_at`. Routine syncs stay + * hash-gated and cheap; a full resync re-hydrates every incident (one show call + * plus its notes and timeline pages each) so note-only and timeline-only changes + * are picked up. That cost is paid on every full resync, not once. + * + * `supportsIncrementalSync` is deliberately absent for the same reason at the + * listing level: PagerDuty's `since`/`until` filter incident *creation* time and + * the REST API exposes no modified-since filter, so an incremental listing would + * never surface a status change or resolution on an incident created before the + * window — an incident synced while triggered would stay triggered forever. */ + rehydrateOnFullSync: true, configFields: [ { diff --git a/apps/sim/connectors/pagerduty/pagerduty.ts b/apps/sim/connectors/pagerduty/pagerduty.ts index dab4a6e7034..f122f7d30ae 100644 --- a/apps/sim/connectors/pagerduty/pagerduty.ts +++ b/apps/sim/connectors/pagerduty/pagerduty.ts @@ -136,7 +136,8 @@ function buildHeaders(accessToken: string): Record { * `updated_at` is bumped whenever the incident is modified, so the hash is stable * between the list stub and `getDocument`. Notes and log entries are child * resources: PagerDuty does not guarantee they bump the parent's `updated_at`, so - * a full resync is the way to pick up note-only changes. + * the connector sets `rehydrateOnFullSync` and an explicit full resync is what + * picks up note-only changes. */ function buildContentHash(incident: PagerDutyIncident): string { return `pagerduty:${incident.id}:${incident.updated_at ?? ''}` @@ -175,6 +176,36 @@ function buildMetadata(incident: PagerDutyIncident): IncidentMetadata { } } +/** Depth beyond which nested `custom_details` are rendered as a single JSON line. */ +const MAX_DETAIL_DEPTH = 4 + +/** + * Flattens a `custom_details` payload into `dotted.path: value` lines. + * + * Events API alerts routinely nest their `custom_details` (a `metadata` object, a + * list of affected hosts), so dropping non-primitive values would discard the most + * specific part of the incident. Recursion is depth-bounded, and anything deeper + * is emitted as compact JSON rather than lost. + */ +function flattenDetails(value: unknown, path: string, depth: number, lines: string[]): void { + if (value == null) return + if (typeof value !== 'object') { + lines.push(path ? `${path}: ${String(value)}` : String(value)) + return + } + if (depth >= MAX_DETAIL_DEPTH) { + lines.push(`${path}: ${JSON.stringify(value)}`) + return + } + if (Array.isArray(value)) { + value.forEach((item, index) => flattenDetails(item, `${path}[${index}]`, depth + 1, lines)) + return + } + for (const [key, nested] of Object.entries(value as Record)) { + flattenDetails(nested, path ? `${path}.${key}` : key, depth + 1, lines) + } +} + /** * Renders the incident body details, which arrive either as a plain/HTML string * (Incident Creation API) or as a structured object (Events API payloads). @@ -186,10 +217,7 @@ function renderBodyDetails(details: unknown): string | undefined { } if (details && typeof details === 'object') { const lines: string[] = [] - for (const [key, value] of Object.entries(details as Record)) { - if (value == null || typeof value === 'object') continue - lines.push(`${key}: ${String(value)}`) - } + flattenDetails(details, '', 0, lines) return lines.length > 0 ? lines.join('\n') : undefined } return undefined @@ -209,11 +237,36 @@ function incidentToStub(incident: PagerDutyIncident): ExternalDocument | null { } } +/** + * Outcome of a child-resource fetch. + * + * `failed` separates a hard failure from "PagerDuty answered, and this incident + * genuinely has no accessible sub-resource" (403/404 — an ability the account + * lacks). The latter is a permanent, complete answer and is folded into the + * document as an empty section; a hard failure means the rendered content is + * missing sections that do exist, and since `contentHash` is keyed on the + * incident's `updated_at` it would never change again — the partial document + * would be frozen in place. The caller therefore fails the document so the next + * sync retries it. + */ +interface ChildFetch { + items: T[] + failed: boolean +} + +/** 403/404 are terminal answers about access, not transient transport failures. */ +function isPermanentlyUnavailable(status: number): boolean { + return status === 403 || status === 404 +} + /** * Fetches every note on an incident. `GET /incidents/{id}/notes` takes no * pagination parameters and returns the full set in one response. */ -async function fetchNotes(accessToken: string, incidentId: string): Promise { +async function fetchNotes( + accessToken: string, + incidentId: string +): Promise> { try { const response = await fetchWithRetry( `${PAGERDUTY_API_BASE}/incidents/${encodeURIComponent(incidentId)}/notes`, @@ -225,17 +278,17 @@ async function fetchNotes(accessToken: string, incidentId: string): Promise { +): Promise> { const incidentId = incident.id as string const entries: PagerDutyLogEntry[] = [] let offset = 0 let bounded = Boolean(incident.created_at) let truncated = false + let failed = false try { while (entries.length < MAX_LOG_ENTRIES) { @@ -294,6 +348,7 @@ async function fetchLogEntries( incidentId, status: response.status, }) + failed = !isPermanentlyUnavailable(response.status) break } @@ -313,16 +368,17 @@ async function fetchLogEntries( incidentId, error: toError(error).message, }) + failed = true } - if (truncated || entries.length > MAX_LOG_ENTRIES) { + if (truncated) { logger.warn('Truncated PagerDuty incident timeline at the per-document cap', { incidentId, cap: MAX_LOG_ENTRIES, }) } - return entries.slice(0, MAX_LOG_ENTRIES) + return { items: entries.slice(0, MAX_LOG_ENTRIES), failed } } /** @@ -601,40 +657,49 @@ export const pagerdutyConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null + if (!externalId) return null - const incident = await fetchIncident(accessToken, externalId) - if (!incident?.id) return null + /** Only a deleted incident (404/410) resolves to `null`; `fetchIncident` throws otherwise. */ + const incident = await fetchIncident(accessToken, externalId) + if (!incident?.id) return null - const [notes, logEntries] = await Promise.all([ - fetchNotes(accessToken, incident.id), - fetchLogEntries(accessToken, incident), - ]) + const [notes, logEntries] = await Promise.all([ + fetchNotes(accessToken, incident.id), + fetchLogEntries(accessToken, incident), + ]) - const content = formatIncidentContent(incident, notes, logEntries) - if (!content.trim()) { - logger.info('Skipping PagerDuty incident with no indexable content', { externalId }) - return null - } - - return { - externalId: incident.id, - title: buildTitle(incident), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: incident.html_url || undefined, - contentHash: buildContentHash(incident), - metadata: { ...buildMetadata(incident) }, - } - } catch (error) { - logger.warn('Failed to get PagerDuty incident', { + /** + * Indexing the incident without its notes or timeline would freeze the gap + * in place: the hash is derived from `updated_at`, which a retry cannot + * change, so the document would never be re-hydrated. Throwing (rather than + * resolving `null`) makes the sync engine record a visible failed document + * and retry it, instead of dropping the incident with no counter. + */ + if (notes.failed || logEntries.failed) { + logger.warn('PagerDuty incident had an incomplete sub-resource fetch', { externalId, - error: toError(error).message, + notesFailed: notes.failed, + logEntriesFailed: logEntries.failed, }) + throw new Error(`Incomplete sub-resource fetch for PagerDuty incident ${externalId}`) + } + + const content = formatIncidentContent(incident, notes.items, logEntries.items) + if (!content.trim()) { + logger.info('Skipping PagerDuty incident with no indexable content', { externalId }) return null } + + return { + externalId: incident.id, + title: buildTitle(incident), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: incident.html_url || undefined, + contentHash: buildContentHash(incident), + metadata: { ...buildMetadata(incident) }, + } }, validateConfig: async ( diff --git a/apps/sim/connectors/reddit/reddit.test.ts b/apps/sim/connectors/reddit/reddit.test.ts new file mode 100644 index 00000000000..efee0a3707d --- /dev/null +++ b/apps/sim/connectors/reddit/reddit.test.ts @@ -0,0 +1,364 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { redditConnector } from '@/connectors/reddit/reddit' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' + +const ACCESS_TOKEN = 'test-token' + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function requestUrl(callIndex = 0): URL { + const call = mockFetch.mock.calls[callIndex] + if (!call) throw new Error(`No fetch call at index ${callIndex}`) + return new URL(String(call[0])) +} + +function postFixture(id: string, overrides: Record = {}) { + return { + kind: 't3', + data: { + id, + name: `t3_${id}`, + title: `Post ${id}`, + selftext: 'body text', + author: 'alice', + score: 10, + num_comments: 2, + created_utc: 1700000000, + permalink: `/r/testsub/comments/${id}/post_${id}/`, + url: `https://www.reddit.com/r/testsub/comments/${id}/`, + subreddit: 'testsub', + is_self: true, + ...overrides, + }, + } +} + +function listing(children: unknown[], after: string | null) { + return { kind: 'Listing', data: { children, after } } +} + +beforeEach(() => { + vi.stubGlobal('fetch', mockFetch) + mockFetch.mockReset() +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('reddit listDocuments request shape', () => { + it('hits oauth.reddit.com with the listing sort, limit and raw_json', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1')], null))) + + await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'r/testsub', sort: 'new' }) + + const url = requestUrl() + expect(url.origin).toBe('https://oauth.reddit.com') + expect(url.pathname).toBe('/r/testsub/new') + expect(url.searchParams.get('limit')).toBe('100') + expect(url.searchParams.get('raw_json')).toBe('1') + expect(url.searchParams.get('count')).toBe('0') + expect(url.searchParams.get('t')).toBeNull() + }) + + it('sends the time filter only for the top sort', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1')], null))) + + await redditConnector.listDocuments(ACCESS_TOKEN, { + subreddit: 'testsub', + sort: 'top', + timeFilter: 'month', + }) + + expect(requestUrl().searchParams.get('t')).toBe('month') + }) + + it('sends a descriptive User-Agent and bearer token', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([], null))) + + await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + + const headers = mockFetch.mock.calls[0][1].headers as Record + expect(headers.Authorization).toBe(`Bearer ${ACCESS_TOKEN}`) + expect(headers['User-Agent']).toBe(REDDIT_USER_AGENT) + expect(REDDIT_USER_AGENT).toMatch(/^web:[\w.-]+:v[\d.]+ \(\+https:\/\//) + }) + + it('rejects a subreddit that would reshape the request path', async () => { + await expect( + redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: '../api/v1/me' }) + ).rejects.toThrow(/valid subreddit/i) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('falls back to hot for an unrecognized sort', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([], null))) + + await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub', sort: '../about' }) + + expect(requestUrl().pathname).toBe('/r/testsub/hot') + }) +}) + +describe('reddit deletion-reconciliation safety', () => { + it('leaves listingCapped unset when the listing is genuinely exhausted', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1')], null))) + const syncContext: Record = {} + + const result = await redditConnector.listDocuments( + ACCESS_TOKEN, + { subreddit: 'testsub' }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('leaves listingCapped unset when maxPosts lands exactly on exhaustion', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1'), postFixture('a2')], null))) + const syncContext: Record = {} + + await redditConnector.listDocuments( + ACCESS_TOKEN, + { subreddit: 'testsub', maxPosts: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when maxPosts stops a listing that still has posts', async () => { + mockFetch.mockResolvedValue( + jsonResponse(listing([postFixture('a1'), postFixture('a2')], 't3_a2')) + ) + const syncContext: Record = {} + + const result = await redditConnector.listDocuments( + ACCESS_TOKEN, + { subreddit: 'testsub', maxPosts: '2' }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it("flags listingCapped when Reddit's ~1000-item listing ceiling ends pagination", async () => { + const page = Array.from({ length: 100 }, (_, i) => postFixture(`p${i}`)) + mockFetch.mockResolvedValue(jsonResponse(listing(page, null))) + const syncContext: Record = {} + + await redditConnector.listDocuments( + ACCESS_TOKEN, + { subreddit: 'testsub', maxPosts: '5000' }, + 'after:t3_prev:collected:900', + syncContext + ) + + expect(syncContext.totalDocsFetched).toBe(1000) + expect(syncContext.listingCapped).toBe(true) + }) + + it('clamps maxPosts to the listing depth ceiling', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1')], null))) + const syncContext: Record = {} + + const result = await redditConnector.listDocuments( + ACCESS_TOKEN, + { subreddit: 'testsub', maxPosts: '999999' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(1) + expect(syncContext.listingCapped).toBeUndefined() + }) +}) + +describe('reddit document mapping', () => { + it('returns deferred stubs whose hash matches getDocument', async () => { + const post = postFixture('a1') + mockFetch.mockResolvedValue(jsonResponse(listing([post], null))) + + const list = await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + const stub = list.documents[0] + expect(stub.contentDeferred).toBe(true) + expect(stub.content).toBe('') + expect(stub.sourceUrl).toBe(`https://www.reddit.com${post.data.permalink}`) + + mockFetch.mockResolvedValue( + jsonResponse([ + listing([post], null), + listing( + [ + { + kind: 't1', + data: { id: 'c1', author: 'bob', body: 'nice', score: 3, created_utc: 1700000100 }, + }, + ], + null + ), + ]) + ) + + const full = await redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'a1') + expect(full?.contentHash).toBe(stub.contentHash) + expect(full?.contentDeferred).toBe(false) + expect(full?.content).toContain('body text') + expect(full?.content).toContain('[bob | score: 3]: nice') + }) + + it('excludes the fuzzed score but tracks edits and comment count in the hash', async () => { + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1', { score: 999 })], null))) + const scored = await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + + mockFetch.mockResolvedValue(jsonResponse(listing([postFixture('a1', { score: 1 })], null))) + const rescored = await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + expect(rescored.documents[0].contentHash).toBe(scored.documents[0].contentHash) + + mockFetch.mockResolvedValue( + jsonResponse(listing([postFixture('a1', { edited: 1700009999 })], null)) + ) + const edited = await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + expect(edited.documents[0].contentHash).not.toBe(scored.documents[0].contentHash) + + mockFetch.mockResolvedValue( + jsonResponse(listing([postFixture('a1', { num_comments: 77 })], null)) + ) + const commented = await redditConnector.listDocuments(ACCESS_TOKEN, { subreddit: 'testsub' }) + expect(commented.documents[0].contentHash).not.toBe(scored.documents[0].contentHash) + }) + + it('drops removal placeholders and surfaces unexpanded more stubs', async () => { + mockFetch.mockResolvedValue( + jsonResponse([ + listing([postFixture('a1', { selftext: '[removed]' })], null), + listing( + [ + { + kind: 't1', + data: { id: 'c1', author: 'bob', body: '[deleted]', score: 1, created_utc: 1 }, + }, + { + kind: 't1', + data: { id: 'c2', author: 'carol', body: 'real', score: 4, created_utc: 2 }, + }, + { kind: 'more', data: { count: 42 } }, + ], + null + ), + ]) + ) + + const doc = await redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'a1') + expect(doc?.content).not.toContain('[removed]') + expect(doc?.content).not.toContain('[deleted]') + expect(doc?.content).toContain('[carol | score: 4]: real') + expect(doc?.content).toContain('42 further comments not indexed') + }) + + it('reads the post from the first listing of the two-element comments response', async () => { + mockFetch.mockResolvedValue( + jsonResponse([listing([postFixture('a1')], null), listing([], null)]) + ) + + const doc = await redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'a1') + expect(doc?.externalId).toBe('a1') + expect(requestUrl().pathname).toBe('/r/testsub/comments/a1') + expect(requestUrl().searchParams.get('depth')).toBe('1') + }) + + it('returns null when the post is gone', async () => { + mockFetch.mockResolvedValue(jsonResponse({ message: 'Not Found' }, 404)) + + const doc = await redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'gone') + expect(doc).toBeNull() + }) + + it('throws instead of reporting an empty document when the fetch fails', async () => { + mockFetch.mockResolvedValue(jsonResponse({ message: 'Forbidden' }, 403)) + + await expect( + redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'a1') + ).rejects.toThrow(/403/) + }) + + it('caps indexed comments and reports the remainder as omitted', async () => { + const comments = Array.from({ length: 20 }, (_, i) => ({ + kind: 't1', + data: { author: `u${i}`, body: `comment ${i}`, score: i }, + })) + mockFetch.mockResolvedValue( + jsonResponse([listing([postFixture('a1')], null), listing(comments, null)]) + ) + + const doc = await redditConnector.getDocument(ACCESS_TOKEN, { subreddit: 'testsub' }, 'a1') + expect(doc?.content).toContain('Top Comments (15):') + expect(doc?.content).toContain('comment 14') + expect(doc?.content).not.toContain('comment 15') + expect(doc?.content).toContain('(5 further comments not indexed)') + }) + + it('maps every declared tag definition', () => { + const tags = redditConnector.mapTags?.({ + author: 'alice', + score: 10, + commentCount: 2, + flair: 'Discussion', + postDate: '2026-01-01T00:00:00.000Z', + }) + + const declared = redditConnector.tagDefinitions?.map((tag) => tag.id) ?? [] + expect(Object.keys(tags ?? {}).sort()).toEqual([...declared].sort()) + }) +}) + +describe('reddit validateConfig', () => { + it('rejects an invalid subreddit before calling the API', async () => { + const result = await redditConnector.validateConfig(ACCESS_TOKEN, { subreddit: 'a b/c' }) + expect(result.valid).toBe(false) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('checks the about endpoint and rejects private subreddits', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ kind: 't5', data: { display_name: 'testsub', subreddit_type: 'private' } }) + ) + + const result = await redditConnector.validateConfig(ACCESS_TOKEN, { subreddit: 'testsub' }) + expect(requestUrl().pathname).toBe('/r/testsub/about') + expect(result).toEqual({ valid: false, error: 'Subreddit r/testsub is private' }) + }) + + it('maps a 404 to a not-found message', async () => { + mockFetch.mockResolvedValue(jsonResponse({ message: 'Not Found' }, 404)) + + const result = await redditConnector.validateConfig(ACCESS_TOKEN, { subreddit: 'testsub' }) + expect(result.valid).toBe(false) + expect(result.error).toMatch(/not found or is not accessible/) + }) + + it('accepts a public subreddit', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ kind: 't5', data: { display_name: 'testsub', subreddit_type: 'public' } }) + ) + + expect(await redditConnector.validateConfig(ACCESS_TOKEN, { subreddit: 'testsub' })).toEqual({ + valid: true, + }) + }) +}) diff --git a/apps/sim/connectors/reddit/reddit.ts b/apps/sim/connectors/reddit/reddit.ts index 07e8d26770e..cd83f7f59bb 100644 --- a/apps/sim/connectors/reddit/reddit.ts +++ b/apps/sim/connectors/reddit/reddit.ts @@ -4,57 +4,102 @@ import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/document import { DEFAULT_MAX_POSTS, redditConnectorMeta } from '@/connectors/reddit/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' const logger = createLogger('RedditConnector') const REDDIT_API_BASE = 'https://oauth.reddit.com' -const REDDIT_USER_AGENT = 'sim-studio:v1.0.0 (knowledge-connector)' + +/** Max `limit` accepted by Reddit listing endpoints. */ const POSTS_PER_PAGE = 100 + +/** Top-level comments appended to each post's indexed content. */ const COMMENTS_PER_POST = 15 +/** + * Reddit listings stop paginating after ~1000 items regardless of `after`, so a + * listing that ends exactly at the ceiling is truncated by Reddit rather than + * exhausted and must not drive deletion reconciliation. + */ +const REDDIT_LISTING_DEPTH_LIMIT = 1000 + +const VALID_SORTS = new Set(['hot', 'new', 'top', 'rising']) +const VALID_TIME_FILTERS = new Set(['hour', 'day', 'week', 'month', 'year', 'all']) + +/** + * Reddit restricts subreddit names to at most 21 letters, digits and + * underscores. The lower bound is 2 rather than the 3 the creation form + * enforces today, so legacy two-character subreddits (r/de) stay syncable. + */ +const SUBREDDIT_PATTERN = /^[A-Za-z0-9_]{2,21}$/ + +/** Body text Reddit substitutes for content removed by a mod or deleted by its author. */ +const REMOVED_PLACEHOLDERS = new Set(['[removed]', '[deleted]']) + interface RedditPost { kind: string data: { id: string - name: string title: string + /** Raw markdown body. Preferred over `selftext_html` so no HTML stripping is needed. */ selftext: string - selftext_html?: string author: string score: number num_comments: number created_utc: number + /** + * Unix seconds when the post was last edited, `false` when never edited, and + * `true` on posts edited before Reddit started recording the timestamp. + */ + edited?: number | boolean permalink: string url: string link_flair_text?: string subreddit: string is_self: boolean - domain?: string } } interface RedditComment { kind: string data: { - id: string author: string body: string score: number - created_utc: number - replies?: RedditListing | string + } +} + +/** `kind: 'more'` stub standing in for comments the tree endpoint did not expand. */ +interface RedditMore { + kind: string + data: { + count?: number } } interface RedditListing { kind: string data: { - children: (RedditPost | RedditComment)[] + children: (RedditPost | RedditComment | RedditMore)[] after: string | null } } +class RedditApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'RedditApiError' + } +} + /** * Makes an authenticated request to the Reddit API. + * + * `raw_json=1` is always sent: without it Reddit HTML-escapes `selftext` and + * comment `body` (`&`, `<`, `'`), which would be indexed verbatim. */ async function redditApiGet( path: string, @@ -62,8 +107,8 @@ async function redditApiGet( params?: Record, retryOptions?: Parameters[2] ): Promise { - const queryParams = params ? `?${new URLSearchParams(params).toString()}` : '' - const url = `${REDDIT_API_BASE}${path}${queryParams}` + const queryParams = new URLSearchParams({ ...params, raw_json: '1' }) + const url = `${REDDIT_API_BASE}${path}?${queryParams.toString()}` const response = await fetchWithRetry( url, @@ -79,67 +124,99 @@ async function redditApiGet( ) if (!response.ok) { - throw new Error(`Reddit API HTTP error: ${response.status} ${response.statusText}`) + throw new RedditApiError( + `Reddit API HTTP error: ${response.status} ${response.statusText}`, + response.status + ) } return response.json() } /** - * Fetches top-level comments for a post, up to a maximum count. + * Normalizes and validates a user-supplied subreddit name. + * + * The value is interpolated into the API path, so anything outside Reddit's own + * name charset is rejected rather than escaped — a slash would otherwise + * redirect the request to a different endpoint. */ -async function fetchPostComments( - accessToken: string, - subreddit: string, - postId: string, - maxComments: number -): Promise { - try { - const data = (await redditApiGet(`/r/${subreddit}/comments/${postId}`, accessToken, { - limit: String(maxComments), - depth: '1', - sort: 'top', - })) as RedditListing[] - - if (!Array.isArray(data) || data.length < 2) return [] - - return extractComments(data[1], maxComments) - } catch (error) { - logger.warn('Failed to fetch comments for post', { - postId, - error: toError(error).message, - }) - return [] +function normalizeSubreddit(value: unknown): string | null { + if (typeof value !== 'string') return null + const name = value.trim().replace(/^\/+/, '').replace(/^r\//i, '').replace(/\/+$/, '') + return SUBREDDIT_PATTERN.test(name) ? name : null +} + +/** + * Resolves sort and time filter from source config, rejecting values outside the + * documented sets so neither can reshape the request path or query. + */ +function resolveSortConfig(sourceConfig: Record): { + sort: string + timeFilter: string +} { + const rawSort = typeof sourceConfig.sort === 'string' ? sourceConfig.sort : '' + const rawTimeFilter = typeof sourceConfig.timeFilter === 'string' ? sourceConfig.timeFilter : '' + return { + sort: VALID_SORTS.has(rawSort) ? rawSort : 'hot', + timeFilter: VALID_TIME_FILTERS.has(rawTimeFilter) ? rawTimeFilter : 'week', } } +/** Resolves the post cap, clamped to Reddit's listing depth ceiling. */ +function resolveMaxPosts(sourceConfig: Record): number { + const raw = sourceConfig.maxPosts ? Number(sourceConfig.maxPosts) : DEFAULT_MAX_POSTS + if (!Number.isFinite(raw) || raw <= 0) return DEFAULT_MAX_POSTS + return Math.min(Math.floor(raw), REDDIT_LISTING_DEPTH_LIMIT) +} + +/** Strips Reddit's removal placeholders so they are never indexed as content. */ +function cleanBody(text: string | undefined): string { + const trimmed = text?.trim() ?? '' + return REMOVED_PLACEHOLDERS.has(trimmed) ? '' : trimmed +} + /** * Extracts formatted comment strings from a Reddit comment listing. + * + * `kind: 'more'` children are stubs the tree endpoint did not expand (they need + * a separate `/api/morechildren` call). They are deliberately not expanded, so + * their hidden count is returned and surfaced in the document instead of being + * dropped silently. */ -function extractComments(commentListing: RedditListing, maxComments: number): string[] { +function extractComments( + commentListing: RedditListing, + maxComments: number +): { comments: string[]; omitted: number } { const comments: string[] = [] + let omitted = 0 for (const child of commentListing.data.children) { + if (child.kind === 'more') { + omitted += (child as RedditMore).data.count ?? 0 + continue + } if (child.kind !== 't1') continue const comment = child as RedditComment - if (!comment.data.body || comment.data.author === 'AutoModerator') continue - comments.push(`[${comment.data.author} | score: ${comment.data.score}]: ${comment.data.body}`) - if (comments.length >= maxComments) break + const body = cleanBody(comment.data.body) + if (!body || comment.data.author === 'AutoModerator') continue + if (comments.length >= maxComments) { + omitted += 1 + continue + } + comments.push(`[${comment.data.author} | score: ${comment.data.score}]: ${body}`) } - return comments + return { comments, omitted } } /** * Formats a Reddit post with its comments into a document content string. - * When `prefetchedComments` is provided, uses those directly instead of fetching. */ -async function formatPostContent( - accessToken: string, +function formatPostContent( post: RedditPost['data'], - maxComments: number, - prefetchedComments?: string[] -): Promise { + comments: string[], + omittedComments: number +): string { const lines: string[] = [] lines.push(`# ${post.title}`) @@ -155,23 +232,22 @@ async function formatPostContent( } lines.push('') - if (post.selftext) { - lines.push(post.selftext) + const body = cleanBody(post.selftext) + if (body) { + lines.push(body) lines.push('') } - if (maxComments > 0) { - const comments = - prefetchedComments ?? - (await fetchPostComments(accessToken, post.subreddit, post.id, maxComments)) - if (comments.length > 0) { - lines.push('---') - lines.push(`Top Comments (${comments.length}):`) + if (comments.length > 0) { + lines.push('---') + lines.push(`Top Comments (${comments.length}):`) + lines.push('') + for (const comment of comments) { + lines.push(comment) lines.push('') - for (const comment of comments) { - lines.push(comment) - lines.push('') - } + } + if (omittedComments > 0) { + lines.push(`(${omittedComments} further comments not indexed)`) } } @@ -179,7 +255,45 @@ async function formatPostContent( } /** - * Fetches posts from a subreddit listing endpoint, handling pagination. + * Change-detection hash for a post. + * + * Derived only from fields present in the listing so `listDocuments` and + * `getDocument` produce the same value. `score` is deliberately excluded — + * Reddit fuzzes vote counts, so including it would re-index every post on every + * sync. `num_comments` is included because the indexed content embeds comments. + */ +function postContentHash(post: RedditPost['data']): string { + const revision = typeof post.edited === 'number' ? post.edited : post.created_utc + return `reddit:${post.id}:${revision}:${post.num_comments}` +} + +/** + * Builds the lightweight listing stub for a post. Shared by `listDocuments` and + * `getDocument` so `contentHash` and `metadata` can never drift between them. + */ +function postToStub(post: RedditPost['data']): ExternalDocument { + return { + externalId: post.id, + title: post.title || 'Untitled', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: `https://www.reddit.com${post.permalink}`, + contentHash: postContentHash(post), + metadata: { + author: post.author, + score: post.score, + commentCount: post.num_comments, + flair: post.link_flair_text ?? undefined, + postDate: new Date(post.created_utc * 1000).toISOString(), + subreddit: post.subreddit, + }, + } +} + +/** + * Fetches posts from a subreddit listing endpoint, requesting further pages + * until `maxPosts` posts are collected or Reddit stops handing out cursors. */ async function fetchSubredditPosts( accessToken: string, @@ -187,6 +301,7 @@ async function fetchSubredditPosts( sort: string, timeFilter: string, maxPosts: number, + alreadyCollected: number, afterCursor?: string ): Promise<{ posts: RedditPost['data'][]; after: string | null }> { const allPosts: RedditPost['data'][] = [] @@ -194,7 +309,10 @@ async function fetchSubredditPosts( while (allPosts.length < maxPosts) { const limit = Math.min(POSTS_PER_PAGE, maxPosts - allPosts.length) - const params: Record = { limit: String(limit) } + const params: Record = { + limit: String(limit), + count: String(alreadyCollected + allPosts.length), + } if (after) { params.after = after @@ -210,7 +328,7 @@ async function fetchSubredditPosts( params )) as RedditListing - const children = data.data.children as RedditPost[] + const children = data.data.children if (children.length === 0) { after = null break @@ -218,7 +336,7 @@ async function fetchSubredditPosts( for (const child of children) { if (child.kind === 't3') { - allPosts.push(child.data) + allPosts.push((child as RedditPost).data) } } @@ -226,19 +344,7 @@ async function fetchSubredditPosts( if (!after) break } - return { posts: allPosts.slice(0, maxPosts), after } -} - -/** - * Resolves sort and time filter from source config. - */ -function resolveSortConfig(sourceConfig: Record): { - sort: string - timeFilter: string -} { - const sort = (sourceConfig.sort as string) || 'hot' - const timeFilter = (sourceConfig.timeFilter as string) || 'week' - return { sort, timeFilter } + return { posts: allPosts, after } } export const redditConnector: ConnectorConfig = { @@ -247,15 +353,16 @@ export const redditConnector: ConnectorConfig = { listDocuments: async ( accessToken: string, sourceConfig: Record, - cursor?: string + cursor?: string, + syncContext?: Record ): Promise => { - const subreddit = (sourceConfig.subreddit as string)?.trim().replace(/^r\//, '') + const subreddit = normalizeSubreddit(sourceConfig.subreddit) if (!subreddit) { - throw new Error('Subreddit is required') + throw new Error('A valid subreddit name is required') } const { sort, timeFilter } = resolveSortConfig(sourceConfig) - const maxPosts = sourceConfig.maxPosts ? Number(sourceConfig.maxPosts) : DEFAULT_MAX_POSTS + const maxPosts = resolveMaxPosts(sourceConfig) logger.info('Syncing Reddit subreddit', { subreddit, sort, timeFilter, maxPosts }) @@ -279,29 +386,43 @@ export const redditConnector: ConnectorConfig = { sort, timeFilter, pageBatchSize, + collectedSoFar, afterToken ) - const documents: ExternalDocument[] = posts.map((post) => ({ - externalId: post.id, - title: post.title, - content: '', - contentDeferred: true, - mimeType: 'text/plain', - sourceUrl: `https://www.reddit.com${post.permalink}`, - contentHash: `reddit:${post.id}:${post.created_utc}`, - metadata: { - author: post.author, - score: post.score, - commentCount: post.num_comments, - flair: post.link_flair_text ?? undefined, - postDate: new Date(post.created_utc * 1000).toISOString(), - subreddit: post.subreddit, - }, - })) + const documents: ExternalDocument[] = posts.map(postToStub) const totalCollected = collectedSoFar + documents.length - const hasMore = after !== null && totalCollected < maxPosts + const sourceHasMore = after !== null + const hitCap = totalCollected >= maxPosts + const hasMore = !hitCap && sourceHasMore + + /** + * The sync engine hard-deletes every stored document absent from a full + * listing, so it must be told whenever this listing is incomplete: + * - the `maxPosts` cap stopped us while Reddit still had posts to give + * - Reddit's own ~1000-item listing ceiling ended pagination early + * + * A cap that lands exactly on an exhausted listing is NOT flagged: that + * listing is complete, and flagging it would strand removed posts in the + * knowledge base forever. Neither `sort` nor `timeFilter` is flagged — + * those are intentional scope filters, not truncation. + */ + if (syncContext) { + syncContext.totalDocsFetched = totalCollected + const truncatedByCap = hitCap && sourceHasMore + const truncatedByDepthCeiling = !sourceHasMore && totalCollected >= REDDIT_LISTING_DEPTH_LIMIT + if (truncatedByCap || truncatedByDepthCeiling) { + logger.info('Reddit listing truncated; deletion reconciliation suppressed', { + subreddit, + totalCollected, + maxPosts, + truncatedByCap, + truncatedByDepthCeiling, + }) + syncContext.listingCapped = true + } + } return { documents, @@ -315,10 +436,15 @@ export const redditConnector: ConnectorConfig = { sourceConfig: Record, externalId: string ): Promise => { - const subreddit = (sourceConfig.subreddit as string)?.trim().replace(/^r\//, '') + const subreddit = normalizeSubreddit(sourceConfig.subreddit) if (!subreddit) return null try { + /** + * `/r/{sub}/comments/{id}` returns a two-element array: the first listing + * holds the post itself, the second the comment tree. `depth=1` keeps the + * tree to top-level comments. + */ const data = (await redditApiGet(`/r/${subreddit}/comments/${externalId}`, accessToken, { limit: String(COMMENTS_PER_POST), depth: '1', @@ -327,38 +453,36 @@ export const redditConnector: ConnectorConfig = { if (!Array.isArray(data) || data.length === 0) return null - const postListing = data[0] - const postChildren = postListing.data.children as RedditPost[] - if (postChildren.length === 0) return null + const postChildren = data[0]?.data?.children ?? [] + const postChild = postChildren.find((child) => child.kind === 't3') as RedditPost | undefined + if (!postChild) return null - const post = postChildren[0].data - const comments = - data.length >= 2 ? extractComments(data[1] as RedditListing, COMMENTS_PER_POST) : [] - const content = await formatPostContent(accessToken, post, COMMENTS_PER_POST, comments) + const post = postChild.data + const { comments, omitted } = + data.length >= 2 + ? extractComments(data[1], COMMENTS_PER_POST) + : { comments: [], omitted: 0 } return { - externalId: post.id, - title: post.title, - content, + ...postToStub(post), + content: formatPostContent(post, comments, omitted), contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://www.reddit.com${post.permalink}`, - contentHash: `reddit:${post.id}:${post.created_utc}`, - metadata: { - author: post.author, - score: post.score, - commentCount: post.num_comments, - flair: post.link_flair_text ?? undefined, - postDate: new Date(post.created_utc * 1000).toISOString(), - subreddit: post.subreddit, - }, } } catch (error) { + /** + * Only a deleted post (404) resolves to `null`. Everything else — a 403 on + * a subreddit that went private, a 5xx, a network failure — is rethrown so + * the sync engine records a visible failed document instead of silently + * dropping the post from the run. + */ + if (error instanceof RedditApiError && error.status === 404) { + return null + } logger.warn('Failed to get Reddit post document', { externalId, error: toError(error).message, }) - return null + throw error } }, @@ -366,13 +490,14 @@ export const redditConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const subredditInput = (sourceConfig.subreddit as string | undefined) - ?.trim() - .replace(/^r\//, '') + const subreddit = normalizeSubreddit(sourceConfig.subreddit) const maxPosts = sourceConfig.maxPosts as string | undefined - if (!subredditInput) { - return { valid: false, error: 'Subreddit is required' } + if (!subreddit) { + return { + valid: false, + error: 'Subreddit is required and may only contain letters, digits and underscores', + } } if (maxPosts && (Number.isNaN(Number(maxPosts)) || Number(maxPosts) <= 0)) { @@ -381,26 +506,25 @@ export const redditConnector: ConnectorConfig = { try { const data = (await redditApiGet( - `/r/${subredditInput}/about`, + `/r/${subreddit}/about`, accessToken, - {}, + undefined, VALIDATE_RETRY_OPTIONS )) as { kind: string; data: { display_name: string; subreddit_type?: string } } if (data.data?.subreddit_type === 'private') { - return { valid: false, error: `Subreddit r/${subredditInput} is private` } + return { valid: false, error: `Subreddit r/${subreddit} is private` } } return { valid: true } } catch (error) { - const message = getErrorMessage(error, 'Failed to validate configuration') - if (message.includes('404') || message.includes('403')) { + if (error instanceof RedditApiError && (error.status === 404 || error.status === 403)) { return { valid: false, - error: `Subreddit r/${subredditInput} not found or is not accessible`, + error: `Subreddit r/${subreddit} not found or is not accessible`, } } - return { valid: false, error: message } + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } } }, @@ -411,11 +535,11 @@ export const redditConnector: ConnectorConfig = { result.author = metadata.author } - if (typeof metadata.score === 'number') { + if (typeof metadata.score === 'number' && !Number.isNaN(metadata.score)) { result.score = metadata.score } - if (typeof metadata.commentCount === 'number') { + if (typeof metadata.commentCount === 'number' && !Number.isNaN(metadata.commentCount)) { result.commentCount = metadata.commentCount } diff --git a/apps/sim/connectors/registry.server.ts b/apps/sim/connectors/registry.server.ts index 229d9b5331f..f184fd6e77f 100644 --- a/apps/sim/connectors/registry.server.ts +++ b/apps/sim/connectors/registry.server.ts @@ -8,7 +8,6 @@ import { confluenceConnector } from '@/connectors/confluence' import { discordConnector } from '@/connectors/discord' import { docusignConnector } from '@/connectors/docusign' import { dropboxConnector } from '@/connectors/dropbox' -import { evernoteConnector } from '@/connectors/evernote' import { fathomConnector } from '@/connectors/fathom' import { firefliesConnector } from '@/connectors/fireflies' import { githubConnector } from '@/connectors/github' @@ -79,7 +78,6 @@ export const CONNECTOR_REGISTRY: ConnectorRegistry = { discord: discordConnector, docusign: docusignConnector, dropbox: dropboxConnector, - evernote: evernoteConnector, fathom: fathomConnector, fireflies: firefliesConnector, github: githubConnector, diff --git a/apps/sim/connectors/registry.ts b/apps/sim/connectors/registry.ts index 8d46de4c98a..de7f123cc42 100644 --- a/apps/sim/connectors/registry.ts +++ b/apps/sim/connectors/registry.ts @@ -8,7 +8,6 @@ import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import { discordConnectorMeta } from '@/connectors/discord/meta' import { docusignConnectorMeta } from '@/connectors/docusign/meta' import { dropboxConnectorMeta } from '@/connectors/dropbox/meta' -import { evernoteConnectorMeta } from '@/connectors/evernote/meta' import { fathomConnectorMeta } from '@/connectors/fathom/meta' import { firefliesConnectorMeta } from '@/connectors/fireflies/meta' import { githubConnectorMeta } from '@/connectors/github/meta' @@ -79,7 +78,6 @@ export const CONNECTOR_META_REGISTRY: ConnectorMetaRegistry = { discord: discordConnectorMeta, docusign: docusignConnectorMeta, dropbox: dropboxConnectorMeta, - evernote: evernoteConnectorMeta, fathom: fathomConnectorMeta, fireflies: firefliesConnectorMeta, github: githubConnectorMeta, diff --git a/apps/sim/connectors/rootly/rootly.ts b/apps/sim/connectors/rootly/rootly.ts index fd99d1d04b1..a87524df9df 100644 --- a/apps/sim/connectors/rootly/rootly.ts +++ b/apps/sim/connectors/rootly/rootly.ts @@ -14,15 +14,42 @@ const PAGE_SIZE = 50 /** Cap on timeline events appended to a document to keep content bounded. */ const MAX_TIMELINE_EVENTS = 200 /** - * JSON:API relationships to embed inline within each incident's `attributes`. - * Rootly omits these unless requested via `include`, so both the list (stub) and - * detail requests pass them to ensure tag metadata is identical on either path. - * Scoped to exactly the relationships this connector reads — `environments`, - * `services`, and `groups` (Rootly's API token for teams) — to avoid fetching - * unused relationship payloads on every incident. + * Relationship names passed as `include` on incident requests — `environments`, + * `services`, and `groups` (Rootly's API token for teams), the only ones this + * connector reads. Rootly's incident schema embeds them inside `attributes`, so + * the include only affects the sideloaded `included[]`; it is sent on both the + * list and detail requests so neither path can serialize the relationship + * attributes differently and make the stub's tags drift from the hydrated + * document's. */ const INCIDENT_INCLUDE = 'environments,services,groups' +/** + * Detail-only include. `incident_post_mortem` sideloads the incident's + * retrospective so `getDocument` can append its body to the indexed content + * without a second round trip. + */ +const INCIDENT_DETAIL_INCLUDE = `${INCIDENT_INCLUDE},incident_post_mortem` + +/** JSON:API resource `type` of a Rootly retrospective in `included[]`. */ +const POST_MORTEM_TYPE = 'incident_post_mortems' + +/** + * Deterministic sort keys. Rootly paginates with `page[number]`, so an unsorted + * listing can reorder between page requests and silently drop incidents — and a + * full-sync listing that drops a document makes the sync engine hard-delete it. + * + * Full syncs sort by `created_at`, which never changes, so page boundaries are + * fixed for the whole walk. Incremental syncs have no immutable key available — + * they are filtered on `updated_at`, the very column that moves — so they sort + * `updated_at` ascending, which pushes a record touched mid-sync ahead of the + * cursor rather than behind it. Its own update is therefore still seen; the + * residual risk is the one-position shift that displacement causes further down + * the listing, which page-number paging cannot fully avoid either way. + */ +const FULL_SYNC_SORT = 'created_at' +const INCREMENTAL_SORT = 'updated_at' + /** * JSON:API named-resource entry as embedded directly inside incident * `attributes` for relationships (environments, services, etc.). Each entry @@ -103,6 +130,13 @@ interface RootlyEventResource { attributes?: RootlyEventAttributes } +/** A sideloaded JSON:API resource from the top-level `included[]` array. */ +interface RootlyIncludedResource { + id?: string + type?: string + attributes?: Record +} + /** JSON:API list envelope shared by incidents and events list endpoints. */ interface RootlyListResponse { data?: T[] @@ -110,12 +144,16 @@ interface RootlyListResponse { next?: string | null } meta?: { + next_page?: number | null + total_pages?: number | null + current_page?: number | null total_count?: number } } interface RootlyResourceResponse { data?: T + included?: RootlyIncludedResource[] } /** @@ -212,6 +250,22 @@ function buildSourceUrl(attrs: RootlyIncidentAttributes): string | undefined { return attrs.url || attrs.short_url || undefined } +/** + * Determines whether another page exists. + * + * `meta.next_page` is the documented per-page indicator — nullable, and null on + * the last page — so it decides whenever it is present. `links.next` (also + * documented nullable) is only consulted when the envelope carries no + * `meta.next_page` at all. + */ +function hasNextPage(body: RootlyListResponse, pageItemCount: number): boolean { + if (pageItemCount === 0) return false + const nextPage = body.meta?.next_page + if (nextPage != null) return Number(nextPage) > 0 + if (body.meta && 'next_page' in body.meta) return false + return Boolean(body.links?.next) +} + /** * Fetches the incident timeline events, following JSON:API pagination until * exhausted or the event cap is reached. Returns an empty array on any failure @@ -246,7 +300,7 @@ async function fetchTimelineEvents( if (event.attributes) events.push(event.attributes) } - if (!body.links?.next || pageEvents.length === 0) break + if (!hasNextPage(body, pageEvents.length)) break pageNumber += 1 } } catch (error) { @@ -256,9 +310,37 @@ async function fetchTimelineEvents( }) } + if (events.length > MAX_TIMELINE_EVENTS) { + logger.warn('Truncating Rootly incident timeline', { + incidentId, + fetched: events.length, + kept: MAX_TIMELINE_EVENTS, + }) + } + return events.slice(0, MAX_TIMELINE_EVENTS) } +/** + * Extracts the retrospective body sideloaded via `include=incident_post_mortem`. + * Both `title` and `content` are optional in the sideloaded resource, so a + * retrospective that carries only one of them still contributes it, and an + * incident without one contributes nothing. + */ +function extractPostMortem( + included: RootlyIncludedResource[] | undefined +): { title?: string; content?: string } | null { + if (!Array.isArray(included)) return null + for (const resource of included) { + if (resource.type !== POST_MORTEM_TYPE) continue + const attrs = resource.attributes ?? {} + const title = typeof attrs.title === 'string' ? attrs.title.trim() : undefined + const content = typeof attrs.content === 'string' ? attrs.content.trim() : undefined + if (title || content) return { title, content } + } + return null +} + /** * Renders an incident plus its timeline into plain-text content. Only sections * with data are emitted, so resolved incidents read cleanly while open ones omit @@ -266,7 +348,8 @@ async function fetchTimelineEvents( */ function formatIncidentContent( attrs: RootlyIncidentAttributes, - events: RootlyEventAttributes[] + events: RootlyEventAttributes[], + postMortem: { title?: string; content?: string } | null ): string { const parts: string[] = [] @@ -289,37 +372,55 @@ function formatIncidentContent( if (attrs.started_at) parts.push(`Started: ${attrs.started_at}`) if (attrs.resolved_at) parts.push(`Resolved: ${attrs.resolved_at}`) - if (attrs.summary?.trim()) { + const summary = attrs.summary?.trim() + if (summary) { parts.push('') parts.push('--- Summary ---') - parts.push(attrs.summary.trim()) + parts.push(summary) } - if (attrs.mitigation_message?.trim()) { + const mitigation = attrs.mitigation_message?.trim() + if (mitigation) { parts.push('') parts.push('--- Mitigation ---') - parts.push(attrs.mitigation_message.trim()) + parts.push(mitigation) } - if (attrs.resolution_message?.trim()) { + const resolution = attrs.resolution_message?.trim() + if (resolution) { parts.push('') parts.push('--- Resolution ---') - parts.push(attrs.resolution_message.trim()) + parts.push(resolution) } - if (attrs.cancellation_message?.trim()) { + const cancellation = attrs.cancellation_message?.trim() + if (cancellation) { parts.push('') parts.push('--- Cancellation ---') - parts.push(attrs.cancellation_message.trim()) + parts.push(cancellation) } - if (events.length > 0) { + const postMortemTitle = postMortem?.title + const postMortemContent = postMortem?.content + if (postMortemTitle || postMortemContent) { parts.push('') - parts.push('--- Timeline ---') + parts.push('--- Retrospective ---') + if (postMortemTitle) parts.push(postMortemTitle) + if (postMortemContent) parts.push(postMortemContent) + } + + if (events.length > 0) { + const timeline: string[] = [] for (const event of events) { - if (!event.event?.trim()) continue + const text = event.event?.trim() + if (!text) continue const when = event.occurred_at || event.created_at - parts.push(when ? `${when}: ${event.event.trim()}` : event.event.trim()) + timeline.push(when ? `${when}: ${text}` : text) + } + if (timeline.length > 0) { + parts.push('') + parts.push('--- Timeline ---') + parts.push(...timeline) } } @@ -390,7 +491,9 @@ export const rootlyConnector: ConnectorConfig = { if (lastSyncAt) { queryParams.set('filter[updated_at][gt]', lastSyncAt.toISOString()) - queryParams.set('sort', '-updated_at') + queryParams.set('sort', INCREMENTAL_SORT) + } else { + queryParams.set('sort', FULL_SYNC_SORT) } const url = `${ROOTLY_API_BASE}/incidents?${queryParams.toString()}` @@ -420,27 +523,56 @@ export const rootlyConnector: ConnectorConfig = { const incidents = body.data ?? [] const allDocuments: ExternalDocument[] = [] + let droppedFromPage = 0 for (const incident of incidents) { const stub = incidentToStub(incident) - if (stub) allDocuments.push(stub) + if (stub) { + allDocuments.push(stub) + } else { + droppedFromPage += 1 + } + } + + /** + * An incident that arrived without an id or attributes is absent from this + * listing even though it still exists in Rootly, so deletion reconciliation + * must not run against a listing that dropped one. + */ + if (droppedFromPage > 0) { + logger.warn('Dropped malformed Rootly incidents from listing', { + pageNumber: startPage, + dropped: droppedFromPage, + }) + if (syncContext) syncContext.listingCapped = true } const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let documents = allDocuments + let truncatedByCap = false if (maxIncidents > 0) { const remaining = Math.max(0, maxIncidents - prevFetched) if (allDocuments.length > remaining) { documents = allDocuments.slice(0, remaining) + truncatedByCap = true } } const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched + + const morePagesAvailable = hasNextPage(body, incidents.length) const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents - if (hitLimit && syncContext) syncContext.listingCapped = true - const hasNextLink = Boolean(body.links?.next) - const hasMore = !hitLimit && hasNextLink && incidents.length > 0 + /** + * Only a cap that actually hid incidents truncates the listing. When the cap + * lands exactly on the final page with nothing left behind, the source is + * genuinely exhausted and deletions must still reconcile. + */ + if (hitLimit && (truncatedByCap || morePagesAvailable) && syncContext) { + syncContext.listingCapped = true + } + + const hasMore = !hitLimit && morePagesAvailable return { documents, @@ -454,51 +586,48 @@ export const rootlyConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null + if (!externalId) return null - const url = `${ROOTLY_API_BASE}/incidents/${encodeURIComponent(externalId)}?include=${encodeURIComponent(INCIDENT_INCLUDE)}` - const response = await fetchWithRetry(url, { - method: 'GET', - headers: buildHeaders(accessToken), - }) + const url = `${ROOTLY_API_BASE}/incidents/${encodeURIComponent(externalId)}?include=${encodeURIComponent(INCIDENT_DETAIL_INCLUDE)}` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: buildHeaders(accessToken), + }) - if (!response.ok) { - if (response.status === 404 || response.status === 410) return null - throw new Error(`Failed to fetch Rootly incident: ${response.status}`) - } + /** + * Only a deleted incident (404/410) resolves to `null`. Every other failure + * throws so the sync engine records a visible failed document instead of + * dropping the incident from the run with no counter and no error log. + */ + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch Rootly incident: ${response.status}`) + } - const body = (await response.json()) as RootlyResourceResponse - const resource = body.data - const attrs = resource?.attributes - const id = resource?.id - if (!id || !attrs) return null - - const events = await fetchTimelineEvents(accessToken, id) - const content = formatIncidentContent(attrs, events) - if (!content.trim()) { - logger.info('Skipping Rootly incident with no indexable content', { externalId: id }) - return null - } - const metadata = buildMetadata(attrs) - - return { - externalId: id, - title: attrs.title?.trim() || `Incident ${id}`, - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: buildSourceUrl(attrs), - contentHash: buildContentHash(id, attrs.updated_at), - metadata: { ...metadata }, - } - } catch (error) { - logger.warn('Failed to get Rootly incident', { - externalId, - error: toError(error).message, - }) + const body = (await response.json()) as RootlyResourceResponse + const resource = body.data + const attrs = resource?.attributes + const id = resource?.id + if (!id || !attrs) return null + + const events = await fetchTimelineEvents(accessToken, id) + const content = formatIncidentContent(attrs, events, extractPostMortem(body.included)) + if (!content.trim()) { + logger.info('Skipping Rootly incident with no indexable content', { externalId: id }) return null } + const metadata = buildMetadata(attrs) + + return { + externalId: id, + title: attrs.title?.trim() || `Incident ${id}`, + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(attrs), + contentHash: buildContentHash(id, attrs.updated_at), + metadata: { ...metadata }, + } }, validateConfig: async ( diff --git a/apps/sim/connectors/s3/meta.ts b/apps/sim/connectors/s3/meta.ts index 3d7637354d4..49367964ba4 100644 --- a/apps/sim/connectors/s3/meta.ts +++ b/apps/sim/connectors/s3/meta.ts @@ -6,7 +6,7 @@ export const s3ConnectorMeta: ConnectorMeta = { name: 'Amazon S3', description: 'Sync text-based objects from Amazon S3 or any S3-compatible store (Cloudflare R2, MinIO) into your knowledge base', - version: '1.1.0', + version: '1.2.0', icon: S3Icon, auth: { diff --git a/apps/sim/connectors/s3/s3.ts b/apps/sim/connectors/s3/s3.ts index ced6a92dc16..bb746b02fd1 100644 --- a/apps/sim/connectors/s3/s3.ts +++ b/apps/sim/connectors/s3/s3.ts @@ -2,6 +2,7 @@ import crypto from 'crypto' import { createLogger } from '@sim/logger' import { isLoopbackHostname } from '@sim/security/ssrf' import { getErrorMessage, toError } from '@sim/utils/errors' +import { truncate } from '@sim/utils/string' import { isHosted } from '@/lib/core/config/env-flags' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' @@ -9,6 +10,7 @@ import { s3ConnectorMeta } from '@/connectors/s3/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, + htmlToPlainText, isSkippedDocument, markSkipped, parseTagDate, @@ -27,11 +29,37 @@ const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES /** Number of objects requested per ListObjectsV2 page (S3 caps at 1000). */ const LIST_MAX_KEYS = 1000 +/** + * Extensions whose bytes are rendered markup rather than prose. Their content is + * run through {@link htmlToPlainText} before indexing so the knowledge base + * stores readable text instead of raw tags. `xml` is deliberately absent: its + * element names are meaningful search terms, and flattening would delete them. + */ +const MARKUP_EXTENSIONS = new Set(['html', 'htm']) + +/** + * Accepted bucket-name shape. The bucket is interpolated into the AWS + * virtual-hosted host (`{bucket}.s3.{region}.amazonaws.com`), so a value + * carrying `/`, `@`, `#`, `?` or `:` would relocate the request to an entirely + * different origin and ship the SigV4 `Authorization` header there. This is + * deliberately looser than AWS's own naming rules (S3-compatible stores are not + * bound by them) while still confining the value to characters that cannot + * escape the host component. + */ +const BUCKET_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{1,62}$/ + +/** Accepted region shape — AWS region codes plus Cloudflare R2's `auto`. */ +const REGION_PATTERN = /^[A-Za-z0-9-]{1,64}$/ + /** * Default set of file extensions considered safely text-extractable. Objects * with any other extension (or no extension) are skipped, since their content * cannot be reliably decoded to plain text. Users can override this list via * the `extensions` config field. + * + * `rtf` is deliberately absent: there is no RTF text extractor here, so the + * indexed content would be the control-word source (`{\rtf1\ansi\deff0...`) + * rather than the document's prose. */ const DEFAULT_EXTENSIONS = new Set([ 'txt', @@ -48,7 +76,6 @@ const DEFAULT_EXTENSIONS = new Set([ 'yaml', 'yml', 'log', - 'rtf', ]) /** @@ -194,6 +221,14 @@ function resolveContext(accessToken: string, sourceConfig: Record segment === '.' || segment === '..') +} + /** * Builds the canonical URI for a bucket-level (ListObjectsV2) request. * @@ -470,11 +521,15 @@ async function listObjectsPage( const url = buildUrl(ctx, bucketPath, canonicalQueryString) - const response = await secureFetchWithRetry(url, { method: 'GET', headers }, retryOptions) + const response = await secureFetchWithRetry( + url, + { method: 'GET', headers, stripAuthOnRedirect: true }, + retryOptions + ) if (!response.ok) { const errorText = await response.text() - throw new Error(`S3 ListObjectsV2 failed: ${response.status} ${errorText}`) + throw new Error(`S3 ListObjectsV2 failed: ${response.status} ${truncate(errorText, 500)}`) } const xml = await response.text() @@ -510,7 +565,16 @@ export const s3Connector: ConnectorConfig = { ) const stubs = objects - .filter((entry) => isSupportedKey(entry.key, allowedExtensions) && entry.size > 0) + .filter((entry) => { + if (!isSupportedKey(entry.key, allowedExtensions) || entry.size <= 0) return false + if (hasDotSegment(entry.key)) { + logger.warn('Skipping S3 object whose key contains a dot path segment', { + key: entry.key, + }) + return false + } + return true + }) .map((entry) => stubOrSkipBySize(objectToStub(ctx, entry), entry.size, MAX_FILE_SIZE)) const { documents, indexableCount, capReached } = takeIndexableWithinCap( @@ -542,17 +606,30 @@ export const s3Connector: ConnectorConfig = { const ctx = resolveContext(accessToken, sourceConfig) const key = externalId + if (hasDotSegment(key)) { + throw new Error(`S3 key contains an unaddressable dot path segment: ${key}`) + } + try { const encodedPath = buildObjectPath(ctx, key) const headers = buildSignedHeaders(ctx, 'GET', encodedPath, '') const url = buildUrl(ctx, encodedPath, '') - const response = await secureFetchWithRetry(url, { method: 'GET', headers }) - + const response = await secureFetchWithRetry(url, { + method: 'GET', + headers, + stripAuthOnRedirect: true, + }) + + /** + * Only a deleted object (404) resolves to `null`. Every other failure is + * rethrown below so the sync engine records a visible failed document + * instead of dropping the object with no counter and no error log. + */ if (response.status === 404) return null if (!response.ok) { const errorText = await response.text() - throw new Error(`S3 GetObject failed: ${response.status} ${errorText}`) + throw new Error(`S3 GetObject failed: ${response.status} ${truncate(errorText, 500)}`) } const etag = normalizeEtag(response.headers.get('etag') ?? '') @@ -580,7 +657,8 @@ export const s3Connector: ConnectorConfig = { sizeLimitSkipReason(MAX_FILE_SIZE) ) } - const content = body.toString('utf-8') + const raw = body.toString('utf-8') + const content = MARKUP_EXTENSIONS.has(getExtension(key)) ? htmlToPlainText(raw) : raw if (!content.trim()) return null const entry: S3ObjectEntry = { @@ -594,7 +672,7 @@ export const s3Connector: ConnectorConfig = { return { ...stub, content, contentDeferred: false } } catch (error) { logger.warn('Failed to get S3 object', { key, error: toError(error).message }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/salesforce/meta.ts b/apps/sim/connectors/salesforce/meta.ts index f52e7043ca6..bf9b4e0fa97 100644 --- a/apps/sim/connectors/salesforce/meta.ts +++ b/apps/sim/connectors/salesforce/meta.ts @@ -8,7 +8,18 @@ export const salesforceConnectorMeta: ConnectorMeta = { version: '1.0.0', icon: SalesforceIcon, - auth: { mode: 'oauth', provider: 'salesforce', requiredScopes: ['api', 'refresh_token'] }, + /** + * `openid` is listed alongside `api` because the connector resolves the org's + * instance URL from `/services/oauth2/userinfo`, Salesforce's OpenID Connect + * UserInfo endpoint. Salesforce does not document a required scope for it, so + * this mirrors the scope set the `salesforce` OAuth provider already requests + * rather than asserting a contract the docs do not state. + */ + auth: { + mode: 'oauth', + provider: 'salesforce', + requiredScopes: ['api', 'refresh_token', 'openid'], + }, configFields: [ { @@ -23,6 +34,15 @@ export const salesforceConnectorMeta: ConnectorMeta = { { label: 'Opportunities', id: 'Opportunity' }, ], }, + { + id: 'articleLanguage', + title: 'Article Language', + type: 'short-input', + required: false, + placeholder: 'e.g. en_US (default: en_US)', + description: + 'Knowledge Articles only. Article queries are pinned to one language, so only articles in this language are synced.', + }, { id: 'maxRecords', title: 'Max Records', diff --git a/apps/sim/connectors/salesforce/salesforce.ts b/apps/sim/connectors/salesforce/salesforce.ts index c5dcce56c65..ef075fc004c 100644 --- a/apps/sim/connectors/salesforce/salesforce.ts +++ b/apps/sim/connectors/salesforce/salesforce.ts @@ -20,8 +20,33 @@ const logger = createLogger('SalesforceConnector') */ const USERINFO_HOSTS = Object.values(SALESFORCE_LOGIN_HOSTS).map((host) => `https://${host}`) const USERINFO_PATH = '/services/oauth2/userinfo' -const API_VERSION = 'v62.0' -const PAGE_SIZE = 200 + +/** + * REST API version, bare (no `v` prefix). The identity/userinfo payload returns + * `urls.rest` as `https://host/services/data/v{version}/`, so the placeholder is + * substituted with the bare number — prefixing it here would yield `vv62.0/`. + */ +const API_VERSION = '62.0' + +/** Matches an ISO language / locale code (`en`, `en_US`). */ +const LANGUAGE_CODE_REGEX = /^[a-z]{2}(_[A-Z]{2})?$/ + +const DEFAULT_ARTICLE_LANGUAGE = 'en_US' + +/** + * Reads the Knowledge article language from config, rejecting anything that is + * not a plain locale code. The value is interpolated into SOQL, so validating + * against this allowlist — rather than escaping — keeps the query injection-free. + */ +function resolveArticleLanguage(sourceConfig: Record): string { + const raw = typeof sourceConfig.articleLanguage === 'string' ? sourceConfig.articleLanguage : '' + const trimmed = raw.trim() + if (!trimmed) return DEFAULT_ARTICLE_LANGUAGE + if (!LANGUAGE_CODE_REGEX.test(trimmed)) { + throw new Error(`Invalid Salesforce article language: ${trimmed}`) + } + return trimmed +} /** SOQL field lists per object type. */ const OBJECT_FIELDS: Record = { @@ -46,11 +71,23 @@ const OBJECT_FIELDS: Record = { ], } as const -/** SOQL WHERE clause additions per object type. */ -const OBJECT_WHERE: Record = { - KnowledgeArticleVersion: - " WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='en_US'", -} as const +/** + * SOQL WHERE clause additions per object type. + * + * KnowledgeArticleVersion is not freely queryable: Salesforce requires article + * queries to "specify either the PublishStatus or the Id field in the WHERE + * clause", so `PublishStatus='Online'` is mandatory rather than an optional + * narrowing, and the docs further advise filtering on a single PublishStatus + * value. `Language` is only conditionally required from API v47.0 onward + * ("you can filter queries on Knowledge article versions with or without + * Language depending on what you are querying"), so it is kept — pinned to one + * user-selectable locale — rather than relying on that hedge holding for the + * abstract KnowledgeArticleVersion view. + */ +function buildWhereClause(objectType: string, language: string): string { + if (objectType !== 'KnowledgeArticleVersion') return '' + return ` WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='${language}'` +} /** * Result of a userinfo lookup: either the parsed payload + the auth host that @@ -117,6 +154,23 @@ async function fetchUserinfo( return { ok: false, status: lastStatus, errorText: lastErrorText } } +/** + * Substitutes the `{version}` placeholder in the identity payload's `urls.rest` + * and guarantees a single trailing slash. Salesforce documents the value as + * `https://host/services/data/v{version}/`, so only the bare version number is + * substituted. + */ +function normalizeRestUrl(rawRestUrl: string | undefined): string | undefined { + if (!rawRestUrl) return undefined + const substituted = rawRestUrl.replace('{version}', API_VERSION) + return substituted.endsWith('/') ? substituted : `${substituted}/` +} + +/** Org origin (`https://host`) for a resolved REST base URL. */ +function toOrigin(restUrl: string): string { + return new URL(restUrl).origin +} + /** * Resolves the Salesforce instance REST URL from the userinfo endpoint. * Caches the result in syncContext to avoid repeated calls. @@ -137,14 +191,12 @@ async function resolveInstanceUrl( } const urls = result.data.urls as Record | undefined - let restUrl = urls?.rest + const restUrl = normalizeRestUrl(urls?.rest) if (!restUrl) { throw new Error('Salesforce userinfo response did not include a REST URL') } - restUrl = restUrl.replace('{version}', API_VERSION) - if (syncContext) { syncContext.instanceUrl = restUrl } @@ -236,7 +288,7 @@ function recordToStub( const id = record.Id as string const title = buildRecordTitle(objectType, record) const lastModified = (record.LastModifiedDate as string) || '' - const baseUrl = instanceUrl.replace(`/services/data/${API_VERSION}/`, '') + const baseUrl = toOrigin(instanceUrl) return { externalId: id, @@ -293,11 +345,17 @@ export const salesforceConnector: ConnectorConfig = { let url: string if (cursor) { - const baseUrl = instanceUrl.replace(`/services/data/${API_VERSION}/`, '') - url = `${baseUrl}${cursor}` + url = `${toOrigin(instanceUrl)}${cursor}` } else { - const whereClause = OBJECT_WHERE[objectType] || '' - const soql = `SELECT ${fields.join(',')} FROM ${objectType}${whereClause} ORDER BY LastModifiedDate DESC LIMIT ${PAGE_SIZE}` + const whereClause = buildWhereClause(objectType, resolveArticleLanguage(sourceConfig)) + /** + * No SOQL `LIMIT`: it bounds the total result set rather than the batch, + * so it would end the sync after a single page. Paging is driven by + * `nextRecordsUrl` over Salesforce's default 2,000-record query batch, + * which is also the documented maximum, so `Sforce-Query-Options` would + * have nothing to raise. + */ + const soql = `SELECT ${fields.join(',')} FROM ${objectType}${whereClause} ORDER BY LastModifiedDate DESC` url = `${instanceUrl}query?q=${encodeURIComponent(soql)}` } @@ -329,10 +387,12 @@ export const salesforceConnector: ConnectorConfig = { ) const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 + let droppedByCap = false if (maxRecords > 0) { - const remaining = maxRecords - previouslyFetched + const remaining = Math.max(0, maxRecords - previouslyFetched) if (documents.length > remaining) { documents.splice(remaining) + droppedByCap = true } } @@ -343,6 +403,15 @@ export const salesforceConnector: ConnectorConfig = { const hasMore = Boolean(nextRecordsUrl) && (maxRecords <= 0 || totalFetched < maxRecords) + /** + * The listing stops short of the source while records remain, so deletion + * reconciliation must not run — it hard-deletes every stored document that + * the capped listing omitted. + */ + if (syncContext && (droppedByCap || (Boolean(nextRecordsUrl) && !hasMore))) { + syncContext.listingCapped = true + } + return { documents, nextCursor: hasMore ? nextRecordsUrl : undefined, @@ -414,6 +483,16 @@ export const salesforceConnector: ConnectorConfig = { return { valid: false, error: 'Max records must be a positive number' } } + let language: string + try { + language = resolveArticleLanguage(sourceConfig) + } catch { + return { + valid: false, + error: 'Article language must be a locale code such as en_US', + } + } + try { const userinfoResult = await fetchUserinfo(accessToken, VALIDATE_RETRY_OPTIONS) @@ -425,15 +504,18 @@ export const salesforceConnector: ConnectorConfig = { } const urls = userinfoResult.data.urls as Record | undefined - let restUrl = urls?.rest + const restUrl = normalizeRestUrl(urls?.rest) if (!restUrl) { return { valid: false, error: 'Could not resolve Salesforce instance URL' } } - restUrl = restUrl.replace('{version}', API_VERSION) - - const soql = `SELECT Id FROM ${objectType} LIMIT 1` + /** + * The object's mandatory `PublishStatus` filter has to be present here too: + * an unfiltered `SELECT Id FROM KnowledgeArticleVersion` is rejected by + * Salesforce, which would fail validation for a correctly configured org. + */ + const soql = `SELECT Id FROM ${objectType}${buildWhereClause(objectType, language)} LIMIT 1` const queryUrl = `${restUrl}query?q=${encodeURIComponent(soql)}` const queryResponse = await fetchWithRetry( diff --git a/apps/sim/connectors/sentry/meta.ts b/apps/sim/connectors/sentry/meta.ts index 1e778a38c8c..a619f26b7a8 100644 --- a/apps/sim/connectors/sentry/meta.ts +++ b/apps/sim/connectors/sentry/meta.ts @@ -86,7 +86,8 @@ export const sentryConnectorMeta: ConnectorMeta = { { label: 'Last 24 hours', id: '24h' }, { label: 'Last 14 days', id: '14d' }, ], - description: 'Time window for the issue stats Sentry computes on the project issues list.', + description: + 'Time window for the per-issue event stats Sentry computes on the issues list. It does not change which issues are synced.', }, { id: 'maxIssues', diff --git a/apps/sim/connectors/sentry/sentry.ts b/apps/sim/connectors/sentry/sentry.ts index d4828390702..50229c73edb 100644 --- a/apps/sim/connectors/sentry/sentry.ts +++ b/apps/sim/connectors/sentry/sentry.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_QUERY, sentryConnectorMeta } from '@/connectors/sentry/meta' @@ -12,11 +12,17 @@ const DEFAULT_HOST = 'sentry.io' const ISSUES_PER_PAGE = 100 /** - * Allowed `statsPeriod` values for the project issues list endpoint. Sentry's - * project issues endpoint only honors `24h` (default) or `14d` for its timeline - * stats; an empty value disables the stats window. Other periods (e.g. `90d`) - * are accepted by the organization issues endpoint but not this one, so they are - * rejected during validation to avoid a silently-ignored filter. + * Allowed values for the per-issue stats window. + * + * On the organization issues endpoint the parameter that selects the timeline + * Sentry computes per issue is `groupStatsPeriod`, whose documented choices are + * `''`, `24h`, `14d`, and `auto`; anything else is rejected with + * `Invalid stats_period`. The similarly-named `statsPeriod` is a different + * parameter on this endpoint — it is the query's *date range* and would filter + * issues out of the listing entirely — so it is deliberately never sent. + * + * `auto` is not offered because it derives the window from an explicit date + * range this connector does not set. */ const ALLOWED_STATS_PERIODS = new Set(['24h', '14d']) @@ -389,21 +395,41 @@ export const sentryConnector: ConnectorConfig = { throw new Error('Organization and project slugs are required') } + const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 + const remaining = + maxIssues > 0 ? Math.max(0, maxIssues - prevFetched) : Number.POSITIVE_INFINITY + /* - * Uses the project issues list endpoint - * `/api/0/projects/{org}/{project}/issues/`. This endpoint is deprecated in favor of - * `/api/0/organizations/{org}/issues/?project=`, but the organization endpoint - * filters by numeric project ID rather than slug — a UX regression for a connector - * keyed on the human-readable project slug. The project endpoint remains functional - * and slug-addressable, so it is retained deliberately for the listing path. Issue - * detail and latest-event fetches use the organization-scoped paths. + * Lists through the organization issues endpoint + * `/api/0/organizations/{org}/issues/?project=`. Sentry marks the + * project-scoped `/api/0/projects/{org}/{project}/issues/` deprecated and names this + * endpoint as its replacement. `project` accepts project slugs as well as numeric ids + * (the docs' own examples include `?project=android&project=javascript-react`), so the + * connector stays keyed on the human-readable slug, and `environment` is a documented + * parameter here. `limit` is capped at 100, which {@link ISSUES_PER_PAGE} matches. Issue detail + * and latest-event fetches already use organization-scoped paths, so the whole + * connector now speaks one path style. + * + * Consequence of the migration: this endpoint always resolves a date range, and + * with no `statsPeriod`/`start`/`end` it defaults to the widest range it accepts + * (90 days). Issues last seen before that window are absent from the listing and + * are reconciled away, which is the same "aged out of the query window" semantic + * the default query already documents. */ - const url = new URL( - `${apiBase}/projects/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/issues/` - ) + const url = new URL(`${apiBase}/organizations/${encodeURIComponent(organization)}/issues/`) + url.searchParams.set('project', project) url.searchParams.set('query', query) - url.searchParams.set('limit', String(ISSUES_PER_PAGE)) - if (statsPeriod) url.searchParams.set('statsPeriod', statsPeriod) + /* + * Sort by first-seen (`new`) rather than Sentry's default last-seen (`date`). + * The cursor encodes a position in the sort key, so a mutable key is unsafe across a + * multi-page sync: an issue whose `lastSeen` advances mid-sync jumps ahead of the + * cursor and is skipped, and a document missing from an otherwise-complete listing is + * hard-deleted by the sync engine's deletion reconciliation. `firstSeen` never changes + * for an existing issue, so paging over it is stable. + */ + url.searchParams.set('sort', 'new') + url.searchParams.set('limit', String(Math.min(ISSUES_PER_PAGE, Math.max(1, remaining)))) + if (statsPeriod) url.searchParams.set('groupStatsPeriod', statsPeriod) if (environment) url.searchParams.set('environment', environment) if (cursor) url.searchParams.set('cursor', cursor) @@ -430,16 +456,9 @@ export const sentryConnector: ConnectorConfig = { const issues = ((await response.json()) as SentryIssue[]).filter((issue) => Boolean(issue.id)) - const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 let documents = issues.map(issueToStub) - let slicedSome = false - if (maxIssues > 0) { - const remaining = Math.max(0, maxIssues - prevFetched) - if (documents.length > remaining) { - slicedSome = true - documents = documents.slice(0, remaining) - } - } + const slicedSome = documents.length > remaining + if (slicedSome) documents = documents.slice(0, remaining) const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched @@ -458,52 +477,50 @@ export const sentryConnector: ConnectorConfig = { } }, + /** + * Hydrates a deferred stub. Only a genuinely gone issue (404/410) resolves to `null`; + * every other failure propagates, because the sync engine counts a rejected hydration + * as a failed document and logs it with its externalId, whereas a `null` return is + * indistinguishable from "deleted at the source" and is dropped silently. + */ getDocument: async ( accessToken: string, sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null + if (!externalId) return null - const { apiBase, organization } = readSourceConfig(sourceConfig) - if (!organization) return null + const { apiBase, organization } = readSourceConfig(sourceConfig) + if (!organization) return null - const url = `${apiBase}/organizations/${encodeURIComponent(organization)}/issues/${encodeURIComponent(externalId)}/` + const url = `${apiBase}/organizations/${encodeURIComponent(organization)}/issues/${encodeURIComponent(externalId)}/` - const response = await secureFetchWithRetry(url, { - method: 'GET', - headers: authHeaders(accessToken), - }) + const response = await secureFetchWithRetry(url, { + method: 'GET', + headers: authHeaders(accessToken), + }) - if (!response.ok) { - if (response.status === 404 || response.status === 410) return null - throw new Error(`Failed to fetch Sentry issue: ${response.status}`) - } + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch Sentry issue: ${response.status}`) + } - const issue = (await response.json()) as SentryIssue - if (!issue?.id) return null - - const event = await fetchLatestEvent(apiBase, organization, accessToken, issue.id) - const content = formatIssueContent(issue, event) - if (!content.trim()) return null - - return { - externalId: issue.id, - title: buildTitle(issue), - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: issue.permalink || undefined, - contentHash: buildContentHash(issue), - metadata: buildMetadata(issue), - } - } catch (error) { - logger.warn('Failed to get Sentry issue', { - externalId, - error: toError(error).message, - }) - return null + const issue = (await response.json()) as SentryIssue + if (!issue?.id) return null + + const event = await fetchLatestEvent(apiBase, organization, accessToken, issue.id) + const content = formatIssueContent(issue, event) + if (!content.trim()) return null + + return { + externalId: issue.id, + title: buildTitle(issue), + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: issue.permalink || undefined, + contentHash: buildContentHash(issue), + metadata: buildMetadata(issue), } }, @@ -568,12 +585,13 @@ export const sentryConnector: ConnectorConfig = { * `listDocuments` and the org-scoped `getDocument`/latest-event hydration — * needs `event:read`. A token scoped to `project:read` only would pass the * first probe yet fail at hydration time, so this second probe forces a - * misconfigured token to fail fast at save time. It is slug-addressable and - * cheap (one issue, no stats window). + * misconfigured token to fail fast at save time. It hits the same endpoint + * `listDocuments` uses, and is cheap (one issue, no stats window). */ const issuesProbeUrl = new URL( - `${apiBase}/projects/${encodeURIComponent(organization)}/${encodeURIComponent(project)}/issues/` + `${apiBase}/organizations/${encodeURIComponent(organization)}/issues/` ) + issuesProbeUrl.searchParams.set('project', project) issuesProbeUrl.searchParams.set('query', DEFAULT_QUERY) issuesProbeUrl.searchParams.set('limit', '1') diff --git a/apps/sim/connectors/servicenow/servicenow.ts b/apps/sim/connectors/servicenow/servicenow.ts index aa81a1dea12..bf223f89f21 100644 --- a/apps/sim/connectors/servicenow/servicenow.ts +++ b/apps/sim/connectors/servicenow/servicenow.ts @@ -142,40 +142,96 @@ interface Incident extends ServiceNowRecord { assigned_to?: ServiceNowField opened_by?: ServiceNowField close_notes?: ServiceNowField - comments_and_work_notes?: ServiceNowField - work_notes?: ServiceNowField resolution_notes?: ServiceNowField } /** * Normalizes and validates the ServiceNow instance URL. * - * Prepends https:// if the scheme is missing, strips trailing slashes, then - * enforces a ServiceNow-owned domain allowlist to prevent SSRF — the instance - * URL is user-controlled and was previously fetched server-side with no - * validation. + * Prepends https:// if the scheme is missing, then enforces a ServiceNow-owned + * domain allowlist to prevent SSRF — the instance URL is user-controlled and was + * previously fetched server-side with no validation. + * + * The result is reduced to the URL's origin. Users routinely paste a full + * console URL (`https://acme.service-now.com/nav_to.do?uri=...`, a `/kb_view.do` + * link, or a trailing slash); every Table API path is built by appending + * `/api/now/table/...`, so any surviving path, query or fragment would produce + * a 404. `URL.origin` also lower-cases the host and drops the default port. */ function resolveServiceNowInstanceUrl(rawUrl: string): string { - let url = (rawUrl ?? '').trim().replace(/\/+$/, '') - if (url && !url.startsWith('https://') && !url.startsWith('http://')) { + let url = (rawUrl ?? '').trim() + if (url && !/^https?:\/\//i.test(url)) { url = `https://${url}` } const validation = validateServiceNowInstanceUrl(url) if (!validation.isValid) { throw new Error(validation.error || 'Invalid instance URL') } - return validation.sanitized ?? url + return new URL(validation.sanitized ?? url).origin } /** - * Builds Basic Auth header from username and API key/password. + * Builds the HTTP Basic auth header from the username and the API key/password. + * + * The username is validated here rather than only in `validateConfig`, because + * `listDocuments`/`getDocument` run against a stored `sourceConfig` that may + * have been written after validation. A missing username would otherwise be + * encoded as the literal `undefined:` and surface as an opaque 401, and a + * username containing `:` cannot be represented in Basic auth at all (RFC 7617 + * splits on the first colon). */ function buildAuthHeader(accessToken: string, sourceConfig: Record): string { - const username = sourceConfig.username as string - const encoded = Buffer.from(`${username}:${accessToken}`).toString('base64') + const username = sourceConfig.username + if (typeof username !== 'string' || !username.trim()) { + throw new Error('ServiceNow username is required') + } + if (username.includes(':')) { + throw new Error('ServiceNow username cannot contain a colon') + } + const encoded = Buffer.from(`${username.trim()}:${accessToken}`).toString('base64') return `Basic ${encoded}` } +/** + * Coerces the user-supplied `maxItems` into a usable positive integer. + * + * A non-numeric value would otherwise make `maxItems` `NaN`, which poisons every + * downstream comparison: `sysparm_limit` becomes `NaN`, `resultCount >= limit` + * is `false` so `nextOffset` is never produced, and — worst — the cap check is + * gated on `nextOffset`, so `listingCapped` is never set and the sync engine + * hard-deletes every document missing from the broken listing. + */ +function resolveMaxItems(value: unknown): number { + const parsed = Math.floor(Number(value)) + if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_MAX_ITEMS + return parsed +} + +/** The `YYYY-MM-DD HH:mm:ss` shape a glide_date_time takes in its raw form. */ +const SERVICENOW_DATETIME_PATTERN = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})$/ + +/** + * Parses a ServiceNow raw datetime (`sys_updated_on`) into a `Date`. + * + * Raw glide_date_time values come back as `YYYY-MM-DD HH:mm:ss` in **UTC** (only + * `display_value` is shifted into the API user's timezone), and every read here + * goes through {@link rawValue}, which prefers `value`. `new Date()` treats that + * space-separated, zone-less form as *local* time, so passing it straight to + * `parseTagDate` skews the tag by the host's UTC offset. Appending the explicit + * `Z` fixes it; anything not matching the ServiceNow shape falls back to the + * shared parser. + */ +function parseServiceNowDate(value: unknown): Date | undefined { + if (typeof value === 'string') { + const match = SERVICENOW_DATETIME_PATTERN.exec(value.trim()) + if (match) { + const date = new Date(`${match[1]}T${match[2]}Z`) + return Number.isNaN(date.getTime()) ? undefined : date + } + } + return parseTagDate(value) +} + /** * Calls the ServiceNow Table API. */ @@ -533,7 +589,7 @@ export const servicenowConnector: ConnectorConfig = { ): Promise => { const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string) const contentType = (sourceConfig.contentType as string) || 'kb_knowledge' - const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS + const maxItems = resolveMaxItems(sourceConfig.maxItems) const authHeader = buildAuthHeader(accessToken, sourceConfig) const offset = cursor ? Number(cursor) : 0 @@ -564,6 +620,7 @@ export const servicenowConnector: ConnectorConfig = { sysparm_query: query, sysparm_fields: fields, sysparm_display_value: 'all', + sysparm_exclude_reference_link: 'true', } logger.info('Fetching ServiceNow records', { @@ -618,9 +675,17 @@ export const servicenowConnector: ConnectorConfig = { * A full page landing exactly on the cap is ambiguous: `nextOffset` is set * whenever a page comes back full, so it cannot distinguish "more rows * follow" from "the table ended on a page boundary". `X-Total-Count` - * resolves it when present; when the header is absent the ambiguity is - * resolved conservatively (assume truncated), since over-flagging only - * defers a purge whereas under-flagging deletes live documents. + * resolves it when present. The Table API documents that header only as + * "Total count of records returned by the query", without stating that it + * ignores `sysparm_limit`/`sysparm_offset`; sibling APIs on the same platform + * state the stronger semantic outright — the Case API describes it as "the + * total number of records matching the request when the `sysparm_limit` or + * `sysparm_offset` query parameters are specified". When the header is absent + * the ambiguity resolves conservatively (assume truncated), since + * over-flagging only defers a purge whereas under-flagging deletes live + * documents. Note the asymmetry: that fallback covers only an *absent* + * header. A page-scoped count would equal the page size, never exceed the + * cap, and so suppress the flag — the one way this check can under-flag. */ if (nextOffset !== undefined && !hasMore && syncContext) { if (totalCount === undefined || totalCount > maxItems) { @@ -674,29 +739,26 @@ export const servicenowConnector: ConnectorConfig = { const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string) - try { - const record = await serviceNowApiGetById(instanceUrl, tableName, externalId, authHeader, { - sysparm_fields: fields, - sysparm_display_value: 'all', - }) - - if (!record || !isServiceNowRecord(record)) { - return null - } - - const doc = isKB - ? kbArticleToDocument(record, instanceUrl) - : incidentToDocument(record, instanceUrl) + /** + * `serviceNowApiGetById` resolves `null` only for a 404 (or an empty result); + * every other failure throws, so the sync engine records a visible failed + * document instead of dropping the record with no counter and no error log. + */ + const record = await serviceNowApiGetById(instanceUrl, tableName, externalId, authHeader, { + sysparm_fields: fields, + sysparm_display_value: 'all', + sysparm_exclude_reference_link: 'true', + }) - return doc.content.trim() ? doc : null - } catch (error) { - logger.warn('Failed to get ServiceNow document', { - externalId, - table: tableName, - error: toError(error).message, - }) + if (!record || !isServiceNowRecord(record)) { return null } + + const doc = isKB + ? kbArticleToDocument(record, instanceUrl) + : incidentToDocument(record, instanceUrl) + + return doc.content.trim() ? doc : null }, validateConfig: async ( @@ -724,17 +786,18 @@ export const servicenowConnector: ConnectorConfig = { return { valid: false, error: 'Max items must be a positive number' } } - let normalizedUrl: string - try { - normalizedUrl = resolveServiceNowInstanceUrl(instanceUrl) - } catch (error) { - return { valid: false, error: toError(error).message } - } - - const authHeader = buildAuthHeader(accessToken, sourceConfig) const tableName = contentType === 'kb_knowledge' ? 'kb_knowledge' : 'incident' + /** + * `resolveServiceNowInstanceUrl` and `buildAuthHeader` both reject malformed + * config by throwing, so they run inside the same try as the probe — outside + * it, a rejected instance URL or a colon-bearing username would escape + * `validateConfig` as an exception instead of a readable validation error. + */ try { + const normalizedUrl = resolveServiceNowInstanceUrl(instanceUrl) + const authHeader = buildAuthHeader(accessToken, sourceConfig) + await serviceNowApiGet( normalizedUrl, tableName, @@ -742,6 +805,7 @@ export const servicenowConnector: ConnectorConfig = { { sysparm_limit: '1', sysparm_offset: '0', + sysparm_fields: 'sys_id', }, VALIDATE_RETRY_OPTIONS ) @@ -776,7 +840,7 @@ export const servicenowConnector: ConnectorConfig = { result.author = author } - const lastUpdated = parseTagDate(metadata.lastUpdated) + const lastUpdated = parseServiceNowDate(metadata.lastUpdated) if (lastUpdated) { result.lastUpdated = lastUpdated } diff --git a/apps/sim/connectors/sftp/meta.ts b/apps/sim/connectors/sftp/meta.ts index 29ac8ada690..61cac344cb8 100644 --- a/apps/sim/connectors/sftp/meta.ts +++ b/apps/sim/connectors/sftp/meta.ts @@ -6,7 +6,7 @@ export const sftpConnectorMeta: ConnectorMeta = { name: 'SFTP', description: 'Sync text-based files from a remote SFTP (SSH File Transfer Protocol) directory tree into your knowledge base', - version: '1.0.0', + version: '2.0.0', icon: SftpIcon, auth: { @@ -58,10 +58,10 @@ export const sftpConnectorMeta: ConnectorMeta = { id: 'hostFingerprint', title: 'Host Key Fingerprint', type: 'short-input', - placeholder: 'e.g. SHA256:abc123... (optional)', - required: false, + placeholder: 'e.g. SHA256:abc123...', + required: true, description: - 'Expected SHA-256 host key fingerprint. Get it with "ssh-keyscan -t rsa,ecdsa,ed25519 | ssh-keygen -lf -" and paste the SHA256:... value. If it does not match, the connection is refused before any credential is sent. Leave empty to skip host verification (the server is then trusted on sight).', + 'Required. Expected SHA-256 host key fingerprint, pinned so the server is identified before any credential is sent. Get it with "ssh-keyscan -t rsa,ecdsa,ed25519 | ssh-keygen -lf -" and paste the SHA256:... value. Without a pin, SSH accepts whatever host key answers, so an on-path attacker impersonating the server would be handed your password or private key. A mismatch refuses the connection.', }, { id: 'rootPath', diff --git a/apps/sim/connectors/sftp/sftp.test.ts b/apps/sim/connectors/sftp/sftp.test.ts new file mode 100644 index 00000000000..a8026582bce --- /dev/null +++ b/apps/sim/connectors/sftp/sftp.test.ts @@ -0,0 +1,267 @@ +/** + * @vitest-environment node + * + * Host key verification is mandatory for this connector, so these tests drive + * the real `createSftpConnection` — including the `hostVerifier` it builds — + * against a fake ssh2 `Client` that presents a known host key. ssh2 itself is + * mocked, so the fake reproduces the one contract the verifier depends on: + * it is called with the raw host key blob during `connect`, before any + * credential is used, and a `false` return aborts the connection. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { sftpConnector } from '@/connectors/sftp/sftp' + +const S_IFDIR = 0o040755 +const S_IFREG = 0o100644 + +/** + * Host key the fake server presents, and the fingerprint OpenSSH prints for it. + * The expected values are written out rather than recomputed with `createHash` + * so the test asserts against a fixed digest instead of restating the + * production code's own formula: + * + * node -e "console.log(require('crypto').createHash('sha256') + * .update(Buffer.from('ssh-ed25519 fake host key bytes')) + * .digest('base64').replace(/=+$/, ''))" + */ +const SERVER_HOST_KEY = Buffer.from('ssh-ed25519 fake host key bytes') +const SERVER_FINGERPRINT = 'vavRyluclyEjo81XjQIzVxoqZgtAY47GuKLG6j6aCGM' +const OTHER_FINGERPRINT = 'Gg8WhyW1o7SY1nxHGvP4EuvFvVCSjtSBclGk7qnas5E' + +const { mockValidateDatabaseHost, clientConnects } = vi.hoisted(() => ({ + mockValidateDatabaseHost: vi.fn(), + clientConnects: [] as Array>, +})) + +vi.mock('@/components/icons', () => ({ SftpIcon: () => null })) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mockValidateDatabaseHost, +})) + +/** + * The factory is evaluated before this module's own imports are initialized, so + * it cannot close over an imported binding (`node:events`). The emitter is + * therefore hand-rolled inside the factory. + */ +vi.mock('ssh2', () => { + class FakeClient { + private handlers: Record void>> = {} + + on(event: string, handler: (arg?: unknown) => void) { + ;(this.handlers[event] ??= []).push(handler) + return this + } + + private emit(event: string, arg?: unknown) { + for (const handler of this.handlers[event] ?? []) handler(arg) + } + + connect(config: Record) { + clientConnects.push(config) + setImmediate(() => { + const verifier = config.hostVerifier as ((key: Buffer) => boolean) | undefined + if (verifier && verifier(SERVER_HOST_KEY) === false) { + this.emit('error', new Error('Host denied (verification failed)')) + return + } + this.emit('ready') + }) + return this + } + + sftp(cb: (err: Error | null, sftp: unknown) => void) { + cb(null, fakeSftp) + } + + end() {} + destroy() {} + } + + return { + Client: FakeClient, + utils: { sftp: { STATUS_CODE: { EOF: 1, NO_SUCH_FILE: 2 } } }, + } +}) + +/** Minimal SFTP subsystem: one directory holding one indexable text file. */ +const fakeSftp = { + stat(path: string, cb: (err: Error | null, attrs?: unknown) => void) { + cb(null, { mode: S_IFDIR, size: 0, mtime: 0 }) + }, + lstat(path: string, cb: (err: Error | null, attrs?: unknown) => void) { + cb(null, { mode: S_IFREG, size: 10, mtime: 1_700_000_000 }) + }, + opendir(path: string, cb: (err: Error | null, handle?: Buffer) => void) { + cb(null, Buffer.from(path)) + }, + readdir(handle: Buffer, cb: (err: unknown, list?: unknown) => void) { + const key = handle.toString() + if (readPages.has(key)) { + const eof = new Error('EOF') as Error & { code: number } + eof.code = 1 + cb(eof) + return + } + readPages.add(key) + cb(null, [{ filename: 'a.txt', attrs: { mode: S_IFREG, size: 10, mtime: 1_700_000_000 } }]) + }, + close(_handle: Buffer, cb: () => void) { + cb() + }, +} + +const readPages = new Set() + +const SECRET = 'hunter2' + +function config(overrides: Record = {}) { + return { + host: 'sftp.example.com', + port: 22, + username: 'sftp-user', + rootPath: '/home/sftp-user/docs', + hostFingerprint: `SHA256:${SERVER_FINGERPRINT}`, + ...overrides, + } +} + +beforeEach(() => { + vi.clearAllMocks() + clientConnects.length = 0 + readPages.clear() + mockValidateDatabaseHost.mockResolvedValue({ isValid: true, resolvedIP: '93.184.216.34' }) +}) + +describe('sftp host key fingerprint is required', () => { + const missing: Array<[string, Record]> = [ + ['absent', { hostFingerprint: undefined }], + ['empty string', { hostFingerprint: '' }], + ['whitespace only', { hostFingerprint: ' ' }], + ['bare SHA256 label', { hostFingerprint: 'SHA256:' }], + ['non-string', { hostFingerprint: null }], + ] + + it.each(missing)('validateConfig rejects a %s fingerprint', async (_label, overrides) => { + const result = await sftpConnector.validateConfig!(SECRET, config(overrides)) + + expect(result.valid).toBe(false) + expect(result.error).toMatch(/Host Key Fingerprint is required|not a SHA-256 host key/) + expect(result.error).toContain('ssh-keyscan') + expect(clientConnects).toHaveLength(0) + }) + + it('listDocuments fails closed when a stored config has no fingerprint', async () => { + await expect( + sftpConnector.listDocuments(SECRET, config({ hostFingerprint: undefined })) + ).rejects.toThrow(/Host Key Fingerprint is required/) + expect(clientConnects).toHaveLength(0) + }) + + it('getDocument fails closed when a stored config has no fingerprint', async () => { + await expect( + sftpConnector.getDocument!( + SECRET, + config({ hostFingerprint: '' }), + '/home/sftp-user/docs/a.txt' + ) + ).rejects.toThrow(/Host Key Fingerprint is required/) + expect(clientConnects).toHaveLength(0) + }) + + it('names how to obtain the fingerprint and why it matters', async () => { + const result = await sftpConnector.validateConfig!(SECRET, config({ hostFingerprint: '' })) + + expect(result.error).toContain('ssh-keyscan -t rsa,ecdsa,ed25519 | ssh-keygen -lf -') + expect(result.error).toContain('on-path attacker') + }) + + it('rejects an MD5 fingerprint with a format-specific message', async () => { + const result = await sftpConnector.validateConfig!( + SECRET, + config({ hostFingerprint: 'aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99' }) + ) + + expect(result.valid).toBe(false) + expect(result.error).toContain('not a SHA-256 host key fingerprint') + expect(clientConnects).toHaveLength(0) + }) +}) + +describe('sftp host key verification', () => { + it('accepts a matching fingerprint and installs a hostVerifier', async () => { + const result = await sftpConnector.validateConfig!(SECRET, config()) + + expect(result).toEqual({ valid: true }) + expect(clientConnects).toHaveLength(1) + expect(typeof clientConnects[0].hostVerifier).toBe('function') + }) + + it('syncs with a matching fingerprint', async () => { + const list = await sftpConnector.listDocuments(SECRET, config()) + + expect(list.documents.map((d) => d.externalId)).toEqual(['/home/sftp-user/docs/a.txt']) + }) + + it('rejects a mismatched fingerprint before authenticating', async () => { + const result = await sftpConnector.validateConfig!( + SECRET, + config({ hostFingerprint: `SHA256:${OTHER_FINGERPRINT}` }) + ) + + expect(result.valid).toBe(false) + expect(result.error).toContain('Host key verification failed') + expect(result.error).toContain(SERVER_FINGERPRINT) + expect(result.error).toContain('ssh-keyscan') + }) + + it('does not send the credential in the connect config until the key matches', async () => { + await sftpConnector.validateConfig!(SECRET, config()) + + /** + * ssh2 runs `hostVerifier` during key exchange, before the authentication + * request, so the password living in the connect config is only ever put on + * the wire after the verifier returned true. + */ + const verifier = clientConnects[0].hostVerifier as (key: Buffer) => boolean + expect(verifier(SERVER_HOST_KEY)).toBe(true) + expect(verifier(Buffer.from('an impostor key'))).toBe(false) + }) +}) + +describe('sftp fingerprint format tolerance', () => { + const accepted: Array<[string, string]> = [ + ['with the SHA256: prefix', `SHA256:${SERVER_FINGERPRINT}`], + ['with a lowercase sha256: prefix', `sha256:${SERVER_FINGERPRINT}`], + ['with no prefix', SERVER_FINGERPRINT], + ['with base64 padding', `SHA256:${SERVER_FINGERPRINT}=`], + ['with surrounding whitespace', ` SHA256:${SERVER_FINGERPRINT} `], + [ + 'with an embedded line wrap', + `SHA256:${SERVER_FINGERPRINT.slice(0, 20)}\n${SERVER_FINGERPRINT.slice(20)}`, + ], + ] + + it.each(accepted)('accepts a fingerprint %s', async (_label, fingerprint) => { + const result = await sftpConnector.validateConfig!( + SECRET, + config({ hostFingerprint: fingerprint }) + ) + + expect(result).toEqual({ valid: true }) + }) + + it('is case-sensitive in the base64 body, since base64 is', async () => { + const flipped = SERVER_FINGERPRINT.split('') + .map((c) => (c === c.toLowerCase() ? c.toUpperCase() : c.toLowerCase())) + .join('') + if (flipped === SERVER_FINGERPRINT) return + + const result = await sftpConnector.validateConfig!( + SECRET, + config({ hostFingerprint: `SHA256:${flipped}` }) + ) + + expect(result.valid).toBe(false) + }) +}) diff --git a/apps/sim/connectors/sftp/sftp.ts b/apps/sim/connectors/sftp/sftp.ts index 97d2297450a..f7da687f66b 100644 --- a/apps/sim/connectors/sftp/sftp.ts +++ b/apps/sim/connectors/sftp/sftp.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import type { Attributes, Client, SFTPWrapper } from 'ssh2' +import { type Attributes, type Client, type SFTPWrapper, utils as ssh2Utils } from 'ssh2' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { createSftpConnection, @@ -8,7 +8,6 @@ import { getSftp, isPathSafe, readSftpFileCapped, - sanitizePath, } from '@/app/api/tools/sftp/utils' import { sftpConnectorMeta } from '@/connectors/sftp/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -41,6 +40,18 @@ const MAX_ALLOWED_FILES = 10_000 /** Hard ceiling on `readdir` calls per sync, bounding wide (rather than deep) trees. */ const MAX_DIRECTORIES = 1000 +/** + * Hard ceiling on entries read from a single remote directory. + * + * `SFTPWrapper.readdir(path)` opens a handle and loops until the server reports + * EOF, accumulating every entry into one array, so a directory holding tens of + * millions of names — from a hostile server or merely a huge one — is an + * unbounded allocation inside a single call that none of the walk's other caps + * can interrupt. Reading the handle page by page lets the walk stop at this + * ceiling instead. + */ +const MAX_ENTRIES_PER_DIRECTORY = 20_000 + /** * Hard ceiling on entries emitted by a single walk, counting oversized files. * Oversized files deliberately do not consume the `maxFiles` budget, so without @@ -86,6 +97,11 @@ const BINARY_SNIFF_BYTES = 8192 * File extensions considered safely text-extractable. Anything else (or a file * with no extension) is skipped, since its bytes cannot be reliably decoded to * plain text. Users override this list via the `extensions` config field. + * + * Markup-bearing formats are included only when this connector can flatten them + * (see {@link HTML_EXTENSIONS}). `rtf` is deliberately absent: there is no RTF + * text extractor here, so the indexed content would be the control-word source + * (`{\rtf1\ansi\deff0...`) rather than the document's prose. */ const DEFAULT_EXTENSIONS = new Set([ 'txt', @@ -102,7 +118,6 @@ const DEFAULT_EXTENSIONS = new Set([ 'yaml', 'yml', 'log', - 'rtf', ]) /** Extensions whose content is rendered markup and must be flattened before indexing. */ @@ -136,8 +151,8 @@ interface SftpContext { username: string password?: string privateKey?: string - /** Optional pinned SHA-256 host key fingerprint; empty means no verification. */ - hostFingerprint?: string + /** Pinned SHA-256 host key fingerprint. Required — there is no unverified mode. */ + hostFingerprint: string rootPath: string allowedExtensions: Set maxDepth: number @@ -169,29 +184,45 @@ function resolveBoundedNumber(raw: unknown, fallback: number, max: number): numb /** * Unpadded base64 of a SHA-256 digest — what OpenSSH prints after the `SHA256:` - * prefix (32 digest bytes encode to 43 base64 characters). + * prefix (32 digest bytes encode to 43 base64 characters). Base64 is + * case-significant, so the body is matched as-is; only the `SHA256:` label is + * treated case-insensitively. */ const SHA256_FINGERPRINT_PATTERN = /^[A-Za-z0-9+/]{43}$/ +/** How to obtain the fingerprint, appended to every fingerprint failure. */ +const FINGERPRINT_HOWTO = + 'Get it by running "ssh-keyscan -t rsa,ecdsa,ed25519 | ssh-keygen -lf -" from a trusted network ' + + 'and pasting the SHA256:... value into the Host Key Fingerprint field.' + +/** Message for a source saved without a fingerprint, or with it edited back out. */ +const FINGERPRINT_REQUIRED_MESSAGE = + 'Host Key Fingerprint is required for SFTP sources. Without it, SSH accepts whatever host key ' + + "answers, so an on-path attacker impersonating the server would be handed this source's " + + `password or private key. ${FINGERPRINT_HOWTO}` + /** - * Normalizes and validates the pinned host key fingerprint. Validation matters - * because host verification is opt-in: a value that normalizes to nothing (a - * bare `SHA256:`), or an MD5 fingerprint, would otherwise be dropped and the - * connection would silently fall back to trusting whatever host answers. + * Normalizes and validates the pinned host key fingerprint. + * + * Host verification is mandatory: an absent, blank, or unusable value fails + * closed here rather than falling through to ssh2's default of accepting any + * host key. Tolerant of the shapes the value is pasted in — with or without the + * `SHA256:` label, with or without base64 `=` padding, and with wrapped or + * surrounding whitespace — because a rejected paste that looks correct pushes + * users toward removing the pin. */ -function resolveHostFingerprint(raw: unknown): string | undefined { - if (typeof raw !== 'string') return undefined - const trimmed = raw.trim() - if (!trimmed) return undefined +function resolveHostFingerprint(raw: unknown): string { + const trimmed = typeof raw === 'string' ? raw.trim() : '' + if (!trimmed) throw new Error(FINGERPRINT_REQUIRED_MESSAGE) const normalized = trimmed .replace(/^sha256:/i, '') + .replace(/\s+/g, '') .replace(/=+$/, '') - .trim() if (!SHA256_FINGERPRINT_PATTERN.test(normalized)) { throw new Error( - 'Host key fingerprint must be a SHA-256 fingerprint, e.g. "SHA256:<43 base64 characters>". ' + - 'Get it with "ssh-keyscan -t rsa,ecdsa,ed25519 | ssh-keygen -lf -".' + `"${trimmed}" is not a SHA-256 host key fingerprint. Expected "SHA256:" followed by 43 base64 ` + + `characters (an MD5 "aa:bb:cc:..." fingerprint will not work). ${FINGERPRINT_HOWTO}` ) } return normalized @@ -208,9 +239,18 @@ function getExtension(filePath: string): string { /** * Normalizes a remote path to an absolute, separator-collapsed form without a * trailing slash (the root `/` is preserved). + * + * Deliberately does NOT percent-decode. SFTP paths are opaque byte strings, not + * URLs, so decoding corrupts every legitimate remote name containing a `%` + * followed by two hex digits: a file listed as `report%20final.txt` would be + * requested as `report final.txt` and could never be hydrated. Traversal is + * rejected up front by {@link isPathSafe}, which does check the decoded form, + * and the result is prefix-checked against the configured root, so declining to + * decode here is also the stricter choice — a literal `%2e%2e%2f` stays literal + * on the wire instead of being turned into `../`. */ function normalizeRemotePath(raw: string): string { - const sanitized = sanitizePath(raw) + const sanitized = raw.replace(/\0/g, '').replace(/\\/g, '/').replace(/\/+/g, '/').trim() const absolute = sanitized.startsWith('/') ? sanitized : `/${sanitized}` const trimmed = absolute.replace(/\/+$/, '') return trimmed === '' ? '/' : trimmed @@ -231,6 +271,12 @@ function isWithinRoot(candidate: string, rootPath: string): boolean { * Resolves connection parameters from the connector's sourceConfig and the * decrypted secret (delivered as `accessToken`). The secret is interpreted as a * password or an OpenSSH private key depending on `authMethod`. + * + * Every entry point — `listDocuments`, `getDocument`, and `validateConfig` — + * resolves through here, which is what makes the host key fingerprint enforced + * at sync time and not merely at validation time. A source persisted before the + * fingerprint became required, or edited out of band afterwards, fails its next + * sync instead of silently connecting unverified. */ function resolveContext(accessToken: string, sourceConfig: Record): SftpContext { const host = ((sourceConfig.host as string) ?? '').trim() @@ -273,8 +319,10 @@ function resolveContext(accessToken: string, sourceConfig: Record( ctx: SftpContext, @@ -318,18 +366,87 @@ async function withSftpSession( } } -/** Promise wrapper around `SFTPWrapper.readdir`. */ -function readRemoteDirectory(sftp: SFTPWrapper, directory: string): Promise { +/** SFTP status the server sends once a directory handle has been read to the end. */ +const SFTP_STATUS_EOF = ssh2Utils.sftp.STATUS_CODE.EOF + +/** One bounded read of a remote directory. */ +interface DirectoryListing { + entries: SftpDirEntry[] + /** True when the per-directory ceiling stopped the read before EOF. */ + truncated: boolean +} + +/** Promise wrapper around `SFTPWrapper.opendir`. */ +function openRemoteDirectory(sftp: SFTPWrapper, directory: string): Promise { return new Promise((resolve, reject) => { - sftp.readdir(directory, (err, list) => { + sftp.opendir(directory, (err, handle) => { if (err) reject(err) - else resolve(list) + else resolve(handle) + }) + }) +} + +/** + * Reads the next page from an open directory handle, resolving null at EOF. + * ssh2 surfaces EOF as an error carrying `STATUS_CODE.EOF` rather than as an + * empty list, so it is translated here instead of propagating as a failure. + */ +function readDirectoryPage(sftp: SFTPWrapper, handle: Buffer): Promise { + return new Promise((resolve, reject) => { + sftp.readdir(handle, (err, list) => { + if (err) { + if ((err as Error & { code?: number }).code === SFTP_STATUS_EOF) resolve(null) + else reject(err) + } else { + resolve(list) + } }) }) } -/** True for the SFTP status the server returns when a path no longer exists. */ +/** + * Lists a remote directory a page at a time, stopping at + * {@link MAX_ENTRIES_PER_DIRECTORY}. The handle is always closed, including on + * the error and ceiling paths, so a walk over many directories cannot exhaust + * the server's open-handle budget. + */ +async function readRemoteDirectory( + sftp: SFTPWrapper, + directory: string +): Promise { + const handle = await openRemoteDirectory(sftp, directory) + const entries: SftpDirEntry[] = [] + let truncated = false + + try { + while (!truncated) { + const page = await readDirectoryPage(sftp, handle) + if (page === null) break + + for (const entry of page) { + if (entries.length >= MAX_ENTRIES_PER_DIRECTORY) { + truncated = true + break + } + entries.push(entry) + } + } + } finally { + sftp.close(handle, () => {}) + } + + return { entries, truncated } +} + +/** + * True for the SFTP status the server returns when a path no longer exists. + * + * Prefers the numeric status ssh2 attaches to SFTP failures; the message match + * is the fallback for the paths that surface a re-wrapped `Error` without one. + */ function isNotFoundError(error: unknown): boolean { + const code = (error as { code?: number } | null)?.code + if (code === ssh2Utils.sftp.STATUS_CODE.NO_SUCH_FILE) return true return /no such file|not found|ENOENT/i.test(getErrorMessage(error, '')) } @@ -421,7 +538,13 @@ async function walkTree( let entries: SftpDirEntry[] try { - entries = await readRemoteDirectory(sftp, current.path) + const listing = await readRemoteDirectory(sftp, current.path) + entries = listing.entries + /** + * A directory read that stopped at the per-directory ceiling leaves real + * files unlisted, exactly like the walk-level caps below. + */ + if (listing.truncated) truncated = true directoriesRead += 1 } catch (error) { /** @@ -586,14 +709,19 @@ export const sftpConnector: ConnectorConfig = { ): Promise => { const ctx = resolveContext(accessToken, sourceConfig) + /** + * These two rejections throw rather than resolve `null`. `null` is the + * connector contract's "the document is gone at the source", which the sync + * engine acts on by letting the document be reconciled away; a path this + * connector refuses to address is instead a configuration or tampering + * fault, and must surface as a visible failed row. + */ if (!isPathSafe(externalId)) { - logger.warn('Rejecting SFTP path with traversal sequences', { externalId }) - return null + throw new Error(`Refusing SFTP path with traversal sequences: ${externalId}`) } const remotePath = normalizeRemotePath(externalId) if (!isWithinRoot(remotePath, ctx.rootPath)) { - logger.warn('Rejecting SFTP path outside the configured root', { remotePath }) - return null + throw new Error(`Refusing SFTP path outside the configured root: ${remotePath}`) } return await withSftpSession(ctx, DOCUMENT_TIMEOUT_MS, async (sftp) => { diff --git a/apps/sim/connectors/sharepoint/sharepoint.test.ts b/apps/sim/connectors/sharepoint/sharepoint.test.ts index ccb6165455d..cbefdc63aac 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.test.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.test.ts @@ -15,6 +15,7 @@ import { normalizeSegment, resolveFolderTarget, serverRelativePathFromUrl, + sharepointConnector, } from '@/connectors/sharepoint/sharepoint' const GRAPH = 'https://graph.microsoft.com/v1.0' @@ -332,6 +333,94 @@ describe('resolveFolderTarget', () => { }) }) +const ITEM_SELECT = + 'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdDateTime,createdBy,parentReference' + +/** File-shaped drive item for children listings. */ +function file(id: string, name: string) { + return { + id, + name, + size: 10, + file: { mimeType: 'text/plain' }, + lastModifiedDateTime: '2026-01-01T00:00:00Z', + } +} + +function childrenRoute(driveId: string, folderId: string | null, items: unknown[]) { + const base = folderId + ? `${GRAPH}/drives/${driveId}/items/${folderId}/children` + : `${GRAPH}/drives/${driveId}/root/children` + return { [`${base}?$top=200&$select=${ITEM_SELECT}`]: { body: { value: items } } } +} + +/** Pre-resolved context, so listDocuments goes straight to the children walk. */ +function listContext() { + return { siteId: SITE_ID, siteName: 'Contoso', driveId: DEFAULT_DRIVE_ID } +} + +function list(maxFiles: string | undefined, syncContext: Record) { + return sharepointConnector.listDocuments( + 'token', + { siteUrl: SITE_URL, maxFiles }, + undefined, + syncContext + ) +} + +describe('listDocuments', () => { + it('flags the listing capped when the cap hides items inside the final page', async () => { + mockGraph( + childrenRoute(DEFAULT_DRIVE_ID, null, [ + file('f1', 'a.txt'), + file('f2', 'b.txt'), + file('f3', 'c.txt'), + ]) + ) + const syncContext = listContext() + + const result = await list('2', syncContext) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it('does not flag the listing capped when the cap lands on the last item', async () => { + mockGraph(childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), file('f2', 'b.txt')])) + const syncContext = listContext() + + const result = await list('2', syncContext) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('drains subfolders within a single call instead of one folder per page', async () => { + mockGraph({ + ...childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('sub', 'Sub')]), + ...childrenRoute(DEFAULT_DRIVE_ID, 'sub', [file('f2', 'b.txt')]), + }) + const syncContext = listContext() + + const result = await list(undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('builds a metadata-only contentHash that getDocument can reproduce', async () => { + mockGraph(childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt')])) + + const result = await list(undefined, listContext()) + + expect(result.documents[0].contentHash).toBe('sharepoint:f1:2026-01-01T00:00:00Z') + expect(result.documents[0].contentDeferred).toBe(true) + }) +}) + describe('serverRelativePathFromUrl', () => { it('strips the site prefix from a site-scoped URL', () => { expect( diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index f3d4b99bdf6..ac29865e2bb 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -18,7 +18,8 @@ import { const logger = createLogger('SharePointConnector') -const GRAPH_BASE = 'https://graph.microsoft.com/v1.0' +const GRAPH_API_ORIGIN = 'https://graph.microsoft.com' +const GRAPH_BASE = `${GRAPH_API_ORIGIN}/v1.0` const SUPPORTED_TEXT_EXTENSIONS = new Set([ '.txt', @@ -37,6 +38,21 @@ const SUPPORTED_TEXT_EXTENSIONS = new Set([ const MAX_DOWNLOAD_SIZE = CONNECTOR_MAX_FILE_BYTES +/** + * The exact driveItem fields the stub is built from. Graph returns the full + * driveItem otherwise, which is an order of magnitude larger per item. + */ +const ITEM_SELECT = + 'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdDateTime,createdBy,parentReference' + +/** + * Folder pages listed within a single `listDocuments` call. The sync engine caps + * a sync at a fixed number of `listDocuments` pages, and a depth-first walk needs + * at least one request per folder — draining several folders per call keeps a + * library with thousands of folders from silently truncating its listing. + */ +const MAX_LIST_REQUESTS_PER_CALL = 25 + /** Microsoft Graph drive item shape (subset of fields we use). */ interface DriveItem { id: string @@ -65,6 +81,7 @@ interface Drive { interface DriveListResponse { value: Drive[] + '@odata.nextLink'?: string } /** A configured folder path resolved to a concrete drive and starting folder. */ @@ -86,6 +103,21 @@ function isSupportedTextFile(name: string): boolean { return SUPPORTED_TEXT_EXTENSIONS.has(name.slice(dotIndex).toLowerCase()) } +/** + * Asserts a request URL points at Microsoft Graph before it is followed with the + * bearer token in the `Authorization` header. Several callers pass a + * server-supplied `@odata.nextLink` — one of which round-trips through the sync + * cursor — so an off-origin link would otherwise hand the access token to a third + * party. Mirrors `assertGraphNextPageUrl` used by the Graph tool routes. + */ +function assertGraphUrl(url: string): string { + const parsed = new URL(url.trim()) + if (parsed.origin !== GRAPH_API_ORIGIN) { + throw new Error('Refusing to follow a non-Microsoft Graph URL') + } + return parsed.toString() +} + /** * Issues an authenticated Graph GET. Non-OK responses are returned as-is so * callers can distinguish 404 (not found) from a genuine failure. @@ -96,7 +128,7 @@ function graphGet( retryOptions?: RetryOptions ): Promise { return fetchWithRetry( - url, + assertGraphUrl(url), { method: 'GET', headers: { @@ -114,14 +146,19 @@ function graphGet( * A root site yields an empty server-relative path. */ function splitSiteUrl(siteUrl: string): { hostname: string; serverRelativePath: string } { - const cleaned = siteUrl.replace(/^https?:\/\//, '').replace(/\/+$/, '') + const cleaned = siteUrl + .trim() + .replace(/^https?:\/\//i, '') + .replace(/[?#].*$/, '') + .replace(/\/+$/, '') const firstSlash = cleaned.indexOf('/') if (firstSlash === -1) { return { hostname: cleaned, serverRelativePath: '' } } + const segments = toPathSegments(cleaned.slice(firstSlash)) return { hostname: cleaned.slice(0, firstSlash), - serverRelativePath: cleaned.slice(firstSlash), + serverRelativePath: segments.length > 0 ? `/${segments.join('/')}` : '', } } @@ -168,7 +205,7 @@ async function downloadFileContent( itemId: string, fileName: string ): Promise { - const url = `${GRAPH_BASE}/drives/${driveId}/items/${itemId}/content` + const url = `${GRAPH_BASE}/drives/${driveId}/items/${encodeURIComponent(itemId)}/content` const response = await fetchWithRetry(url, { method: 'GET', @@ -241,8 +278,8 @@ async function listFolderItems( const url = nextLink ?? (folderId - ? `${GRAPH_BASE}/drives/${driveId}/items/${folderId}/children?$top=200` - : `${GRAPH_BASE}/drives/${driveId}/root/children?$top=200`) + ? `${GRAPH_BASE}/drives/${driveId}/items/${folderId}/children?$top=200&$select=${ITEM_SELECT}` + : `${GRAPH_BASE}/drives/${driveId}/root/children?$top=200&$select=${ITEM_SELECT}`) const response = await graphGet(url, accessToken) @@ -268,6 +305,9 @@ const MAX_CHILD_PAGES_PER_SEGMENT = 50 /** Number of sibling names quoted back in a "folder not found" error. */ const MAX_SUGGESTED_NAMES = 25 +/** Bounds the paged document-library listing so a bad cursor cannot spin forever. */ +const MAX_DRIVE_PAGES = 20 + /** * Folds away the differences that make a visually-correct folder name fail * byte-exact path addressing: Unicode composition, invisible characters, and @@ -284,12 +324,20 @@ export function normalizeSegment(value: string): string { .toLowerCase() } -/** Splits a slash-separated path into non-empty, trimmed segments. */ +/** + * Splits a slash-separated path into non-empty, trimmed segments. + * + * Dot segments are dropped rather than passed through. Every segment list here + * ends up concatenated into a Graph URL that `assertGraphUrl` parses with `new + * URL`, which resolves `..` against the Graph base and would silently retarget + * the request at an unrelated Graph resource. SharePoint does not allow an item + * named "." or "..", so nothing addressable is lost. + */ function toPathSegments(path: string): string[] { return path .split('/') .map((segment) => segment.trim()) - .filter(Boolean) + .filter((segment) => segment !== '' && segment !== '.' && segment !== '..') } function encodePathSegments(segments: string[]): string { @@ -557,14 +605,20 @@ async function listSiteDrives( siteId: string, retryOptions?: RetryOptions ): Promise { - const response = await graphGet( - `${GRAPH_BASE}/sites/${siteId}/drives?$select=id,name,webUrl`, - accessToken, - retryOptions - ) - if (!response.ok) return [] - const data = (await response.json()) as DriveListResponse - return data.value ?? [] + const drives: Drive[] = [] + let url = `${GRAPH_BASE}/sites/${siteId}/drives?$select=id,name,webUrl` + + for (let page = 0; page < MAX_DRIVE_PAGES; page++) { + const response = await graphGet(url, accessToken, retryOptions) + if (!response.ok) break + const data = (await response.json()) as DriveListResponse + drives.push(...(data.value ?? [])) + const nextLink = data['@odata.nextLink'] + if (!nextLink) break + url = nextLink + } + + return drives } /** @@ -727,72 +781,87 @@ export const sharepointConnector: ConnectorConfig = { const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0 let totalFetched = (syncContext?.totalDocsFetched as number) ?? 0 - // Process one page of items from the current folder - const data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) - - // Separate files and subfolders - const subfolders: string[] = [] - const files: DriveItem[] = [] - - for (const item of data.value) { - if (item.folder) { - subfolders.push(item.id) - } else if (item.file && isSupportedTextFile(item.name)) { - // Keep oversized files; they are surfaced as skipped (failed) docs below. - files.push(item) + /** Set when the walk stopped for good — either the cap hit or the source ran out. */ + let stopPaging = false + /** Set when the cap truncated a listing that still had items left to list. */ + let cappedWithItemsLeft = false + + for (let request = 0; request < MAX_LIST_REQUESTS_PER_CALL; request++) { + const data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) + + // Separate files and subfolders + const subfolders: string[] = [] + const files: DriveItem[] = [] + + for (const item of data.value) { + if (item.folder) { + subfolders.push(item.id) + } else if (item.file && isSupportedTextFile(item.name)) { + // Keep oversized files; they are surfaced as skipped (failed) docs below. + files.push(item) + } } - } - // Push subfolders onto the stack for depth-first traversal - state.folderStack.push(...subfolders) + // Push subfolders onto the stack for depth-first traversal + state.folderStack.push(...subfolders) - // Convert files to lightweight stubs (no content download). Oversized files are - // kept as skipped stubs but do not consume the max-files cap. - const previouslyFetched = totalFetched - const stubs = files.map((file) => - stubOrSkipBySize(itemToStub(file, siteName), file.size, MAX_DOWNLOAD_SIZE) - ) - const { - documents: pageDocuments, - indexableCount, - capReached, - } = takeIndexableWithinCap(stubs, isSkippedDocument, maxFiles, previouslyFetched) - documents.push(...pageDocuments) + // Convert files to lightweight stubs (no content download). Oversized files are + // kept as skipped stubs but do not consume the max-files cap. + const stubs = files.map((file) => + stubOrSkipBySize(itemToStub(file, siteName), file.size, MAX_DOWNLOAD_SIZE) + ) + const take = takeIndexableWithinCap(stubs, isSkippedDocument, maxFiles, totalFetched) + documents.push(...take.documents) + totalFetched += take.indexableCount + + const nextLink = data['@odata.nextLink'] + + if (take.capReached) { + stopPaging = true + /** + * Only a cap that actually hid items makes the listing partial. When the + * cap coincides with the last item of the last folder the source *is* + * fully listed, and flagging it capped would block deletion + * reconciliation for a complete listing. Items can be left behind in + * this very page (the cap cut it short), in later pages of this folder, + * or in folders still on the stack. + */ + cappedWithItemsLeft = + take.documents.length < stubs.length || Boolean(nextLink) || state.folderStack.length > 0 + break + } - totalFetched += indexableCount + if (nextLink) { + // More pages in the current folder + state.nextLink = nextLink + continue + } - if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitLimit = capReached - if (hitLimit && syncContext) syncContext.listingCapped = true + // Current folder exhausted — move to next folder on the stack + if (state.folderStack.length > 0) { + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue + } - if (hitLimit) { - return { documents, hasMore: false } + stopPaging = true + break } - if (data['@odata.nextLink']) { - // More pages in the current folder - state.nextLink = data['@odata.nextLink'] - return { - documents, - nextCursor: encodeCursor(state), - hasMore: true, - } + if (syncContext) { + syncContext.totalDocsFetched = totalFetched + if (cappedWithItemsLeft) syncContext.listingCapped = true } - // Current folder exhausted — move to next folder on the stack - if (state.folderStack.length > 0) { - const nextFolder = state.folderStack.pop()! - state.currentFolder = nextFolder - state.nextLink = undefined - return { - documents, - nextCursor: encodeCursor(state), - hasMore: true, - } + if (stopPaging) { + return { documents, hasMore: false } } - // Nothing left - return { documents, hasMore: false } + return { + documents, + nextCursor: encodeCursor(state), + hasMore: true, + } }, getDocument: async ( @@ -836,7 +905,7 @@ export const sharepointConnector: ConnectorConfig = { } } - const url = `${GRAPH_BASE}/drives/${driveId}/items/${externalId}` + const url = `${GRAPH_BASE}/drives/${driveId}/items/${encodeURIComponent(externalId)}?$select=${ITEM_SELECT}` const response = await graphGet(url, accessToken) if (!response.ok) { @@ -864,10 +933,15 @@ export const sharepointConnector: ConnectorConfig = { sizeLimitSkipReason(error.limitBytes) ) } + /** + * A transport or Graph failure that survived `fetchWithRetry`. Returning + * `null` would drop the file from the run with no `failed` row and no error + * log; rethrowing lets the sync engine record it per-document. + */ logger.warn(`Failed to fetch content for file: ${item.name} (${item.id})`, { error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/slack/slack.ts b/apps/sim/connectors/slack/slack.ts index e82e9c259a4..00ea03c55be 100644 --- a/apps/sim/connectors/slack/slack.ts +++ b/apps/sim/connectors/slack/slack.ts @@ -70,6 +70,40 @@ interface SlackUser { } } +/** + * Actionable hints for the Slack error codes a sync realistically hits. Without + * them a failed sync surfaces only the raw code (e.g. `not_in_channel`), which + * does not tell the user the fix is to invite the app to the channel. + * Codes are documented per method, e.g. + * https://docs.slack.dev/reference/methods/conversations.history/ + */ +const SLACK_ERROR_HINTS: Record = { + not_in_channel: 'invite the Sim app to this channel', + channel_not_found: 'the channel does not exist or the app cannot see it', + is_archived: 'the channel is archived', + missing_scope: 'the Slack credential is missing a required scope; reconnect it', + invalid_auth: 'the Slack credential is no longer valid; reconnect it', + account_inactive: 'the Slack credential is no longer valid; reconnect it', + token_revoked: 'the Slack credential is no longer valid; reconnect it', + ratelimited: 'Slack rate limit exceeded', +} + +/** + * Error thrown for a Slack `ok: false` envelope, carrying the machine-readable + * `error` code so callers can branch on `channel_not_found` without string + * matching. + */ +class SlackApiError extends Error { + constructor( + readonly code: string, + readonly method: string + ) { + const hint = SLACK_ERROR_HINTS[code] + super(`Slack API error on ${method}: ${code}${hint ? ` — ${hint}` : ''}`) + this.name = 'SlackApiError' + } +} + /** * Calls a Slack Web API method via GET with query params. * Slack returns HTTP 200 even for errors, so we check the `ok` field. @@ -102,13 +136,27 @@ async function slackApiGet( const data = (await response.json()) as Record if (!data.ok) { - const error = (data.error as string) || 'unknown_error' - throw new Error(`Slack API error: ${error}`) + throw new SlackApiError((data.error as string) || 'unknown_error', method) } return data } +/** + * Resolves the configured message window, falling back to the default for + * missing, non-numeric, or non-positive values. + * + * `validateConfig` rejects those inputs, but a config saved before validation + * tightened (or edited out-of-band) would otherwise produce `NaN`/`0` here, + * making `fetchChannelMessages` return zero messages. An empty document is + * dropped from the listing, and the sync engine hard-deletes stored documents + * absent from a listing — silently wiping every indexed channel. + */ +function resolveMaxMessages(value: unknown): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : DEFAULT_MAX_MESSAGES +} + /** * Resolves a user ID to a display name, using a cache stored in syncContext. */ @@ -144,6 +192,16 @@ async function resolveUserName( userId, error: toError(error).message, }) + /** + * Negative-cache only permanently unresolvable users. A deleted user + * (`user_not_found`) can author hundreds of messages, each otherwise + * costing another Tier 4 `users.info` request. Transient failures are not + * cached so a later message can still resolve the real name. + */ + if (syncContext && error instanceof SlackApiError && error.code === 'user_not_found') { + const cache = syncContext[cacheKey] as Record + cache[userId] = userId + } return userId } } @@ -157,7 +215,11 @@ function formatSlackTimestamp(ts: string): string { } /** - * Fetches all messages from a channel, up to a maximum count, handling pagination. + * Fetches messages from a channel, newest first, up to `maxMessages`. + * + * `conversations.history` returns only top-level messages; replies inside a + * thread are served by `conversations.replies` and are therefore NOT indexed — + * threaded discussion content is missing from the synced document. */ async function fetchChannelMessages( accessToken: string, @@ -359,8 +421,16 @@ async function resolveChannel( try { const data = await slackApiGet('conversations.info', accessToken, { channel: trimmed }) return data.channel as SlackChannel - } catch { - // Fall through to name-based search + } catch (error) { + /** + * Only an unknown channel justifies the name-based fallback. Rethrowing + * everything else (auth failures, `missing_scope`, exhausted rate-limit + * retries) avoids walking the full `conversations.list` just to report a + * misleading "Channel not found" for a channel that does exist. + */ + if (!(error instanceof SlackApiError) || error.code !== 'channel_not_found') { + throw error + } } } @@ -496,9 +566,7 @@ export const slackConnector: ConnectorConfig = { throw new Error('At least one channel is required') } - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) logger.info('Syncing Slack channels', { channels: channelInputs, maxMessages }) @@ -564,9 +632,7 @@ export const slackConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { - const maxMessages = sourceConfig.maxMessages - ? Number(sourceConfig.maxMessages) - : DEFAULT_MAX_MESSAGES + const maxMessages = resolveMaxMessages(sourceConfig.maxMessages) try { const data = await slackApiGet('conversations.info', accessToken, { channel: externalId }) @@ -597,11 +663,15 @@ export const slackConnector: ConnectorConfig = { }, } } catch (error) { - logger.warn('Failed to get Slack channel document', { - externalId, - error: toError(error).message, - }) - return null + /** + * `null` means "gone" to the sync engine, so only a deleted channel maps + * to it. Auth, scope and transport failures are rethrown so they surface + * as a failed document rather than a silent drop. + */ + if (error instanceof SlackApiError && error.code === 'channel_not_found') { + return null + } + throw error } }, @@ -639,8 +709,17 @@ export const slackConnector: ConnectorConfig = { { channel: trimmed }, VALIDATE_RETRY_OPTIONS ) - } catch { - return { valid: false, error: `Channel not found: ${input}` } + } catch (error) { + /** + * Only an unknown channel is reported as missing. A scope, auth or + * transport failure falls through to the outer catch and keeps its + * own message — otherwise the user re-picks a channel that exists + * instead of reconnecting the credential. + */ + if (error instanceof SlackApiError && error.code === 'channel_not_found') { + return { valid: false, error: `Channel not found: ${input}` } + } + throw error } } else { nameLookups.push(trimmed) diff --git a/apps/sim/connectors/trello/trello.ts b/apps/sim/connectors/trello/trello.ts index 721e78215df..9874824b278 100644 --- a/apps/sim/connectors/trello/trello.ts +++ b/apps/sim/connectors/trello/trello.ts @@ -26,14 +26,17 @@ const CARD_FIELDS = 'id,name,desc,url,shortUrl,closed,due,dueComplete,dateLastActivity,idList,idBoard,labels,badges' /** - * Maximum cards requested per Trello request. The API caps long collections at - * 1000 results and documents `before`/`since` as the way to page past that cap. - * Neither the ordering of `GET /lists/{id}/cards` nor which 1000 results `limit` - * keeps is documented, so a list that needs a second request is reported as - * capped even though the extra pages are still collected. + * Hard ceiling Trello applies to a long collection: "the Trello API limits you + * to at most 1000 results", with `before`/`since` documented as the way to page + * past it. It is both the page size requested for cards and the size at which + * any cursor-less collection response (boards, lists) is assumed truncated — + * neither the ordering of `GET /lists/{id}/cards` nor which 1000 results `limit` + * keeps is documented, so a collection that reaches the ceiling is reported as + * capped rather than letting deletion reconciliation purge the part of it this + * run could not reach. * @see https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/ */ -const CARD_PAGE_LIMIT = 1000 +const TRELLO_COLLECTION_LIMIT = 1000 /** * Soft per-call document target. Trello has no board-wide card cursor, so the @@ -483,15 +486,28 @@ export const trelloConnector: ConnectorConfig = { const cardFilter = resolveCardFilter(sourceConfig) const state = decodeCursor(cursor) - const boards = - (syncContext?.boards as TrelloBoardRef[] | undefined) ?? - (await resolveBoards(accessToken, sourceConfig)) - if (syncContext) syncContext.boards = boards - const markCapped = () => { if (syncContext) syncContext.listingCapped = true } + const cachedBoards = syncContext?.boards as TrelloBoardRef[] | undefined + const boards = cachedBoards ?? (await resolveBoards(accessToken, sourceConfig)) + if (syncContext) syncContext.boards = boards + + /** + * `GET /members/me/boards` has no cursor, so a response at Trello's + * collection ceiling hides the remaining boards. Their cards would be absent + * from the listing and look deleted, so reconciliation is withheld. Checked + * only on the call that resolved them, since the flag persists on + * `syncContext` for the rest of the run. + */ + if (!cachedBoards && boards.length >= TRELLO_COLLECTION_LIMIT) { + logger.warn('Trello board listing hit the collection ceiling; boards may be missing', { + boardCount: boards.length, + }) + markCapped() + } + const documents: ExternalDocument[] = [] const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 let boardIndex = state.boardIndex @@ -515,7 +531,20 @@ export const trelloConnector: ConnectorConfig = { try { const result = await getCachedLists(accessToken, board.id, cardFilter, syncContext) lists = result.lists - if (result.fetched) requestsUsed += 1 + if (result.fetched) { + requestsUsed += 1 + /** + * `GET /boards/{id}/lists` has no cursor either, so a response at the + * collection ceiling hides the remaining lists and their cards. + */ + if (lists.length >= TRELLO_COLLECTION_LIMIT) { + logger.warn('Trello list listing hit the collection ceiling; lists may be missing', { + boardId: board.id, + listCount: lists.length, + }) + markCapped() + } + } } catch (error) { requestsUsed += 1 logger.warn('Failed to list Trello lists for board', { @@ -548,7 +577,7 @@ export const trelloConnector: ConnectorConfig = { { filter: cardFilter, fields: CARD_FIELDS, - limit: String(CARD_PAGE_LIMIT), + limit: String(TRELLO_COLLECTION_LIMIT), ...(beforeId ? { before: beforeId } : {}), } ) @@ -565,7 +594,7 @@ export const trelloConnector: ConnectorConfig = { } const rawCards = (Array.isArray(cards) ? cards : []).filter((card) => Boolean(card?.id)) - const pageFull = rawCards.length >= CARD_PAGE_LIMIT + const pageFull = rawCards.length >= TRELLO_COLLECTION_LIMIT /** * Trello's `before` bound is a creation date derived from the id, so it is @@ -707,9 +736,25 @@ export const trelloConnector: ConnectorConfig = { }) } - const comments = (Array.isArray(card.actions) ? card.actions : []) - .filter((action) => action?.type === 'commentCard') - .slice(0, COMMENT_LIMIT) + const comments = (Array.isArray(card.actions) ? card.actions : []).filter( + (action) => action?.type === 'commentCard' + ) + + /** + * `actions_limit` caps the nested actions Trello returns, so `card.actions` + * can never hold more than `COMMENT_LIMIT` comments and the shortfall is + * invisible in the array itself. `badges.comments` is the card's true + * comment count and arrives on the same request, so it is what detects the + * truncation. + */ + const totalComments = card.badges?.comments ?? 0 + if (totalComments > COMMENT_LIMIT) { + logger.warn('Trello card comments truncated', { + externalId, + commentLimit: COMMENT_LIMIT, + commentCount: totalComments, + }) + } return { externalId: card.id, @@ -722,12 +767,17 @@ export const trelloConnector: ConnectorConfig = { metadata: cardMetadata(card, boardName, listName), } } catch (error) { + /** + * Only a deleted card resolves to `null`. Any other failure is rethrown so + * the sync engine records a failed row — swallowing it would drop the card + * from the run silently while leaving its stored copy stale. + */ if (error instanceof TrelloApiError && error.status === 404) return null logger.warn('Failed to get Trello card', { externalId, error: toError(error).message, }) - return null + throw toError(error) } }, diff --git a/apps/sim/connectors/typeform/meta.ts b/apps/sim/connectors/typeform/meta.ts index 975d8708fb2..c8f993d212f 100644 --- a/apps/sim/connectors/typeform/meta.ts +++ b/apps/sim/connectors/typeform/meta.ts @@ -39,7 +39,7 @@ export const typeformConnectorMeta: ConnectorMeta = { options: [ { label: 'Completed only', id: 'completed' }, { label: 'Partial & completed', id: 'partial' }, - { label: 'All (including started)', id: 'all' }, + { label: 'All available (partial & completed)', id: 'all' }, ], description: 'Which responses to sync by completion status. Defaults to completed only.', }, @@ -50,7 +50,8 @@ export const typeformConnectorMeta: ConnectorMeta = { required: false, mode: 'advanced', placeholder: 'e.g. 2024-01-01T00:00:00Z', - description: 'Only sync responses submitted on or after this date (ISO 8601, UTC).', + description: + 'Only sync responses on or after this date (ISO 8601, UTC). Compared against submitted_at for completed responses, staged_at for partial, landed_at for started.', }, { id: 'until', @@ -59,7 +60,7 @@ export const typeformConnectorMeta: ConnectorMeta = { required: false, mode: 'advanced', placeholder: 'e.g. 2024-12-31T23:59:59Z', - description: 'Only sync responses submitted on or before this date (ISO 8601, UTC).', + description: 'Only sync responses on or before this date (ISO 8601, UTC).', }, { id: 'query', diff --git a/apps/sim/connectors/typeform/typeform.test.ts b/apps/sim/connectors/typeform/typeform.test.ts new file mode 100644 index 00000000000..973fe9bac0e --- /dev/null +++ b/apps/sim/connectors/typeform/typeform.test.ts @@ -0,0 +1,233 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { typeformConnector } from '@/connectors/typeform/typeform' + +const ACCESS_TOKEN = 'test-token' +const FORM_CONFIG = { formId: 'abc123' } + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const FORM_DEFINITION = { + id: 'abc123', + title: 'Feedback', + fields: [{ id: 'f1', title: 'How was it?' }], + _links: { display: 'https://form.typeform.com/to/abc123' }, +} + +/** Queues the form-definition fetch that always precedes the responses fetch. */ +function mockFormThenResponses(responsesBody: unknown) { + mockFetch + .mockResolvedValueOnce(jsonResponse(FORM_DEFINITION)) + .mockResolvedValueOnce(jsonResponse(responsesBody)) +} + +/** Resolves the URL of the nth (0-indexed) fetch the connector performed. */ +function requestUrl(callIndex = 0): URL { + const call = mockFetch.mock.calls[callIndex] + if (!call) throw new Error(`No fetch call at index ${callIndex}`) + return new URL(String(call[0])) +} + +describe('typeform listDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('sends the response_type filter explicitly rather than relying on the API default', async () => { + mockFormThenResponses({ items: [] }) + + await typeformConnector.listDocuments(ACCESS_TOKEN, FORM_CONFIG) + + expect(requestUrl(1).searchParams.get('response_type')).toBe('completed') + }) + + /** + * Typeform documents only `partial` and `completed` as `response_type` members, + * so `all` must not send an undocumented `started`: an unknown enum member risks + * a 400 that fails the entire sync. + */ + it('requests only the documented response types for the "all" choice', async () => { + mockFormThenResponses({ items: [] }) + + await typeformConnector.listDocuments(ACCESS_TOKEN, { ...FORM_CONFIG, responseType: 'all' }) + + expect(requestUrl(1).searchParams.get('response_type')).toBe('partial,completed') + }) + + it('derives an incremental since filter at the second precision the API documents', async () => { + mockFormThenResponses({ items: [] }) + + await typeformConnector.listDocuments( + ACCESS_TOKEN, + FORM_CONFIG, + undefined, + {}, + new Date('2026-03-20T14:00:59.123Z') + ) + + expect(requestUrl(1).searchParams.get('since')).toBe('2026-03-20T14:00:59Z') + }) + + /** + * A `multi_format` answer is an object carrying the recording plus Typeform's + * generated transcript — not a string. Rendering it directly would index the + * literal text "[object Object]". + */ + it('renders the transcript of a multi_format answer', async () => { + mockFormThenResponses({ + items: [ + { + response_id: 'r1', + token: 't1', + submitted_at: '2026-03-20T14:00:59Z', + answers: [ + { + field: { id: 'f1' }, + type: 'multi_format', + multi_format: { + video_url: 'https://api.typeform.com/video/xyz', + video_transcript: 'It was great', + }, + }, + ], + }, + ], + }) + + const result = await typeformConnector.listDocuments(ACCESS_TOKEN, FORM_CONFIG) + + expect(result.documents[0].content).toContain('How was it?: It was great') + }) + + /** + * Each variable stores its value under the property named by its own `type` + * (`text` or `number`); there is no generic `value` property to read. + */ + it('renders variable values from their type-named property', async () => { + mockFormThenResponses({ + items: [ + { + response_id: 'r1', + token: 't1', + submitted_at: '2026-03-20T14:00:59Z', + answers: [], + variables: [ + { key: 'score', type: 'number', number: 42 }, + { key: 'source', type: 'text', text: 'newsletter' }, + ], + }, + ], + }) + + const result = await typeformConnector.listDocuments(ACCESS_TOKEN, FORM_CONFIG) + + expect(result.documents[0].content).toContain('score: 42') + expect(result.documents[0].content).toContain('source: newsletter') + }) + + it('flags the listing capped only when maxResponses hides responses that still exist', async () => { + mockFormThenResponses({ + items: [ + { response_id: 'r1', token: 't1', submitted_at: '2026-03-20T14:00:59Z' }, + { response_id: 'r2', token: 't2', submitted_at: '2026-03-20T13:00:59Z' }, + ], + }) + + const capped: Record = {} + const result = await typeformConnector.listDocuments( + ACCESS_TOKEN, + { ...FORM_CONFIG, maxResponses: '1' }, + undefined, + capped + ) + + expect(result.documents).toHaveLength(1) + expect(result.hasMore).toBe(false) + expect(capped.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset when the cap lands exactly on source exhaustion', async () => { + mockFormThenResponses({ + items: [{ response_id: 'r1', token: 't1', submitted_at: '2026-03-20T14:00:59Z' }], + }) + + const syncContext: Record = {} + await typeformConnector.listDocuments( + ACCESS_TOKEN, + { ...FORM_CONFIG, maxResponses: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('produces the same contentHash from listDocuments and getDocument', async () => { + const item = { response_id: 'r1', token: 't1', submitted_at: '2026-03-20T14:00:59Z' } + mockFormThenResponses({ items: [item] }) + const syncContext: Record = {} + const listed = await typeformConnector.listDocuments( + ACCESS_TOKEN, + FORM_CONFIG, + undefined, + syncContext + ) + + mockFetch.mockResolvedValueOnce(jsonResponse({ items: [item] })) + const fetched = await typeformConnector.getDocument( + ACCESS_TOKEN, + FORM_CONFIG, + 'r1', + syncContext + ) + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + }) +}) + +describe('typeform getDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('returns null when the response is absent from the result set', async () => { + mockFormThenResponses({ items: [] }) + + await expect( + typeformConnector.getDocument(ACCESS_TOKEN, FORM_CONFIG, 'missing') + ).resolves.toBeNull() + }) + + /** + * Swallowing a server error into `null` would let the sync engine treat a live + * response as deleted, so anything other than a 404 must surface. + */ + it('throws on a server error instead of reporting the response as deleted', async () => { + mockFetch + .mockResolvedValueOnce(jsonResponse(FORM_DEFINITION)) + .mockResolvedValueOnce(jsonResponse({ error: 'boom' }, 500)) + + await expect(typeformConnector.getDocument(ACCESS_TOKEN, FORM_CONFIG, 'r1')).rejects.toThrow( + '500' + ) + }) +}) diff --git a/apps/sim/connectors/typeform/typeform.ts b/apps/sim/connectors/typeform/typeform.ts index 7a12b5c4fdc..c9890b26595 100644 --- a/apps/sim/connectors/typeform/typeform.ts +++ b/apps/sim/connectors/typeform/typeform.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { typeformConnectorMeta } from '@/connectors/typeform/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -12,9 +12,10 @@ const TYPEFORM_API_BASE = 'https://api.typeform.com' const RESPONSES_PER_PAGE = 100 /** - * Allowed `response_type` filter values per the Responses API. `completed` is the - * API default; `all` is a connector-local sentinel that omits the filter so every - * response type (`started`, `partial`, `completed`) is returned. + * Connector-local choices mapped onto the Responses API `response_type` filter. + * The API accepts a comma-separated list of `started`, `partial`, `completed` and + * defaults to `completed` when the parameter is omitted, so `all` must be sent + * explicitly rather than by omission. */ type ResponseTypeChoice = 'completed' | 'partial' | 'all' @@ -56,6 +57,25 @@ interface TypeformAnswer { choice?: { label?: string; other?: string } choices?: { labels?: string[]; other?: string } payment?: { amount?: string; last4?: string; name?: string; success?: boolean } + signature?: { url?: string; type?: string } + multi_format?: { + video_url?: string + video_transcript?: string + audio_url?: string + audio_transcript?: string + } +} + +/** + * A single variable captured with a response. Each entry carries a `key`, a + * `type` of `text` or `number`, and the value under the property named by that + * type — there is no generic `value` property. + */ +interface TypeformVariable { + key?: string + type?: string + text?: string + number?: number } /** @@ -69,7 +89,6 @@ interface TypeformAnswer { interface TypeformResponseItem { response_id?: string token: string - landing_id?: string landed_at?: string submitted_at?: string metadata?: { @@ -79,6 +98,17 @@ interface TypeformResponseItem { } answers?: TypeformAnswer[] | null hidden?: Record | null + variables?: TypeformVariable[] | null +} + +/** + * Formats a Date for Typeform's `since`/`until` filters, which the docs example + * as second-precision ISO 8601 (`2020-03-20T14:00:59`). Dropping the milliseconds + * `toISOString()` emits rounds down, and both filters are inclusive, so the + * window can only widen — never skip a response. + */ +function toTypeformTimestamp(date: Date): string { + return `${date.toISOString().slice(0, 19)}Z` } /** @@ -92,14 +122,24 @@ function getResponseTypeChoice(sourceConfig: Record): ResponseT } /** - * Appends the `response_type` filter to a query string for a given choice. `all` - * omits the parameter so every type is returned; `partial` requests both partial - * and completed so partially-answered submissions are included alongside finished - * ones. + * Appends the `response_type` filter for a given choice. Omitting the parameter + * would fall back to the API default of `completed` only, so every choice is sent + * explicitly. + * + * Typeform documents exactly two members: "It is expected to be passed as a comma + * separated list of values, e.g. `response_type=partial,completed`", defaulting to + * `completed`. There is no documented `started` member — the `sort` docs imply a + * started *state* exists (ordering falls back to `landed_at`), but that does not + * make it a valid filter value, and sending an undocumented member risks a 400 + * that fails the entire sync. `all` therefore requests the widest documented set, + * which is the same as `partial`. */ function appendResponseType(params: URLSearchParams, choice: ResponseTypeChoice): void { - if (choice === 'completed') params.append('response_type', 'completed') - else if (choice === 'partial') params.append('response_type', 'partial,completed') + if (choice === 'partial' || choice === 'all') { + params.append('response_type', 'partial,completed') + } else { + params.append('response_type', 'completed') + } } /** @@ -134,10 +174,42 @@ function renderAnswerValue(answer: TypeformAnswer): string { return parts.join(', ') } case 'payment': - return answer.payment?.amount != null ? String(answer.payment.amount) : '' + return answer.payment?.amount ?? '' + case 'signature': + return answer.signature?.url ?? '' + case 'multi_format': { + /** + * A `multi_format` answer carries an audio or video recording plus the + * transcript Typeform generated for it. The transcript is the indexable + * part; the URL is the fallback when no transcript was produced. + */ + const m = answer.multi_format + return m?.video_transcript || m?.audio_transcript || m?.video_url || m?.audio_url || '' + } default: - return '' + return renderUnknownAnswerValue(answer) + } +} + +/** + * Best-effort rendering for an answer whose `type` this connector does not model. + * Every documented Typeform answer stores its value under a property named by the + * answer's own `type`, either as a scalar or as an object with a `url`/`label`, + * so a new variant degrades to partial content instead of vanishing. + */ +function renderUnknownAnswerValue(answer: TypeformAnswer): string { + const key = answer.type + if (!key) return '' + const value = (answer as Record)[key] + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + return String(value) + } + if (value && typeof value === 'object') { + const { url, label } = value as { url?: unknown; label?: unknown } + if (typeof label === 'string') return label + if (typeof url === 'string') return url } + return '' } /** @@ -181,6 +253,17 @@ function renderResponseContent( } } + const variables = Array.isArray(response.variables) ? response.variables : [] + if (variables.length > 0) { + parts.push('') + parts.push('--- Variables ---') + for (const variable of variables) { + if (!variable?.key) continue + const value = variable.text ?? variable.number + parts.push(`${variable.key}: ${value != null ? String(value) : ''}`) + } + } + return parts.join('\n') } @@ -299,12 +382,15 @@ export const typeformConnector: ConnectorConfig = { /** * `since` from the user config wins; otherwise incremental sync derives it - * from lastSyncAt. `since` narrows the set by submission date while `before` - * (token paging) walks it newest-to-oldest; the two compose — only `sort` is - * mutually exclusive with `before`/`after`, which this connector never sets. + * from lastSyncAt. `since` narrows the set by date while `before` (token + * paging) walks it newest-to-oldest. Which timestamp `since`/`until` filter + * on depends on `response_type`: `submitted_at` when completed responses are + * requested, `staged_at` for partial, `landed_at` otherwise. The docs name + * `since`/`until` and `before`/`after` as the two ways to scope a form with + * more than 1000 responses, so the connector combines both. */ if (since) queryParams.append('since', since) - else if (lastSyncAt) queryParams.append('since', lastSyncAt.toISOString()) + else if (lastSyncAt) queryParams.append('since', toTypeformTimestamp(lastSyncAt)) if (cursor) { queryParams.append('before', cursor) @@ -401,50 +487,53 @@ export const typeformConnector: ConnectorConfig = { syncContext?: Record ): Promise => { const formId = (sourceConfig.formId as string)?.trim() - if (!formId || !externalId) return null - - try { - const form = await getFormDefinition(accessToken, formId, syncContext) - const fieldTitles = buildFieldTitleMap(form) + /** + * A misconfigured source is a failure, not an absent response. Returning + * `null` reads as documented absence, which on an `add` drops the document + * with no counter and no log. `listDocuments` throws on the same condition. + */ + if (!formId) throw new Error('Form ID is required') + if (!externalId) throw new Error('Response ID is required') - /** - * `included_response_ids` filters by `response_id`, matching the externalId - * minted in listDocuments. The configured response_type is forwarded so a - * partial response stays fetchable (the endpoint defaults to completed-only, - * which would otherwise exclude it). - */ - const params = new URLSearchParams() - params.append('included_response_ids', externalId) - appendResponseType(params, getResponseTypeChoice(sourceConfig)) - - const url = `${TYPEFORM_API_BASE}/forms/${encodeURIComponent(formId)}/responses?${params.toString()}` - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + const form = await getFormDefinition(accessToken, formId, syncContext) + const fieldTitles = buildFieldTitleMap(form) - if (!response.ok) { - if (response.status === 404) return null - throw new Error(`Failed to fetch Typeform response ${externalId}: ${response.status}`) - } + /** + * `included_response_ids` filters by `response_id`, matching the externalId + * minted in listDocuments. The configured response_type is forwarded so a + * partial response stays fetchable (the endpoint defaults to completed-only, + * which would otherwise exclude it). + */ + const params = new URLSearchParams() + params.append('included_response_ids', externalId) + appendResponseType(params, getResponseTypeChoice(sourceConfig)) - const data = (await response.json()) as { items?: TypeformResponseItem[] } - const item = Array.isArray(data.items) - ? data.items.find((candidate) => getResponseExternalId(candidate) === externalId) - : undefined - if (!item) return null + const url = `${TYPEFORM_API_BASE}/forms/${encodeURIComponent(formId)}/responses?${params.toString()}` + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) - return responseToDocument(form, item, fieldTitles) - } catch (error) { - logger.warn('Failed to get Typeform response', { - externalId, - error: toError(error).message, - }) - return null + /** + * Only a deleted form/response (404, or an empty match below) resolves to + * `null`. Every other failure throws so the sync engine records a visible + * failed document instead of dropping the response with no counter. + */ + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to fetch Typeform response ${externalId}: ${response.status}`) } + + const data = (await response.json()) as { items?: TypeformResponseItem[] } + const item = Array.isArray(data.items) + ? data.items.find((candidate) => getResponseExternalId(candidate) === externalId) + : undefined + if (!item) return null + + return responseToDocument(form, item, fieldTitles) }, validateConfig: async ( diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 90cb977eb4e..121a0ec99c8 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -62,6 +62,7 @@ import { sentryConnector } from '@/connectors/sentry/sentry' import { typeformConnector } from '@/connectors/typeform/typeform' import { ConnectorFileTooLargeError, + htmlToPlainText, isSkippedDocument, markSkipped, readBodyWithLimit, @@ -1310,3 +1311,57 @@ describe('takeIndexableWithinCap', () => { expect(res.capReached).toBe(true) }) }) + +describe('htmlToPlainText entity decoding', () => { + it('decodes decimal numeric references', () => { + expect(htmlToPlainText('

Sim’s docs – part …

')).toBe( + 'Sim’s docs – part …' + ) + }) + + it('decodes hex numeric references, case-insensitively', () => { + expect(htmlToPlainText('

’–

')).toBe('’–') + }) + + it('decodes astral-plane code points as a surrogate pair', () => { + expect(htmlToPlainText('

😀

')).toBe('\u{1F600}') + }) + + it('still decodes the named entities it always handled', () => { + expect(htmlToPlainText('

<a> "b" 'c' d&e f

')).toBe( + ' "b" \'c\' d&e f' + ) + }) + + it('does not double-decode an escaped entity', () => { + expect(htmlToPlainText('

&#8217;

')).toBe('’') + }) + + it('does not double-decode a numerically escaped ampersand into a named entity', () => { + expect(htmlToPlainText('

&amp; &lt;

')).toBe('& <') + }) + + it('remaps windows-1252 C1 references the way a browser renders them', () => { + expect(htmlToPlainText('

Sim’s “docs” — part …

')).toBe( + 'Sim’s “docs” — part …' + ) + }) + + it('leaves NUL and other control references as literal text', () => { + expect(htmlToPlainText('

a�bcd

')).toBe('a�bcd') + }) + + it('decodes whitespace references and folds them into the whitespace collapse', () => { + expect(htmlToPlainText('

a b

')).toBe('a b') + }) + + it('leaves malformed and out-of-range references as literal text', () => { + expect(htmlToPlainText('

� � &#; &#x;

')).toBe( + '� � &#; &#x;' + ) + }) + + it('leaves an unknown named entity untouched', () => { + expect(htmlToPlainText('

© ¬real;

')).toBe('© ¬real;') + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 49d9c306696..5e04a03619a 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -14,18 +14,123 @@ import type { ExternalDocument } from '@/connectors/types' */ export const CONNECTOR_MAX_FILE_BYTES = KB_DOCUMENT_MAX_BYTES +/** The named entities connector markup actually carries, decoded to their character. */ +const NAMED_ENTITIES: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + nbsp: ' ', +} + +/** + * The C1 range (0x80–0x9F) remapped to the windows-1252 characters browsers + * render, per the HTML standard's numeric character reference table. + * + * Legacy CMS exports — WordPress and Zendesk especially — emit typographic + * punctuation as `’`/`“`/`—`. Those code points are unassigned + * controls in Unicode, so decoding them literally would put invisible garbage + * into the index where the source plainly meant `’`, `“`, `—`. + */ +const WINDOWS_1252_C1 = new Map([ + [0x80, '€'], + [0x82, '‚'], + [0x83, 'ƒ'], + [0x84, '„'], + [0x85, '…'], + [0x86, '†'], + [0x87, '‡'], + [0x88, 'ˆ'], + [0x89, '‰'], + [0x8a, 'Š'], + [0x8b, '‹'], + [0x8c, 'Œ'], + [0x8e, 'Ž'], + [0x91, '‘'], + [0x92, '’'], + [0x93, '“'], + [0x94, '”'], + [0x95, '•'], + [0x96, '–'], + [0x97, '—'], + [0x98, '˜'], + [0x99, '™'], + [0x9a, 'š'], + [0x9b, '›'], + [0x9c, 'œ'], + [0x9e, 'ž'], + [0x9f, 'Ÿ'], +]) + +/** Controls worth decoding — every other control code point stays literal. */ +const DECODABLE_CONTROLS = new Set([0x09, 0x0a, 0x0d]) + +/** + * One alternation covering every reference form, so the whole string is decoded + * in a single left-to-right pass. That ordering is what makes an escaped entity + * safe: `&#8217;` consumes `&` and resumes *after* it, leaving the + * literal text `’` rather than decoding it a second time. + */ +const HTML_ENTITY_PATTERN = /&(?:#[xX]([0-9a-fA-F]+)|#([0-9]+)|(amp|lt|gt|quot|nbsp));/g + +/** + * Resolves a numeric character reference to its character, or returns the + * reference as written when the code point would not survive indexing. + * + * Out-of-range values, lone surrogates, and control characters — notably `�`, + * which Postgres rejects outright in a text column — degrade to literal text so + * malformed source markup never aborts a sync. + */ +function decodeCharacterReference(raw: string, code: number): string { + if (code > 0x10ffff) return raw + if (code >= 0xd800 && code <= 0xdfff) return raw + + const remapped = WINDOWS_1252_C1.get(code) + if (remapped !== undefined) return remapped + + const isControl = code < 0x20 || (code >= 0x7f && code <= 0x9f) + if (isControl && !DECODABLE_CONTROLS.has(code)) return raw + + return String.fromCodePoint(code) +} + /** - * Strips HTML tags from content and decodes common HTML entities. + * Anchored to a tag-name allowlist rather than the looser `<[a-z!/]` shape, + * because {@link htmlToPlainText} both strips tags and collapses all whitespace. + * A false positive therefore does not merely pass text through untouched — it + * deletes the bracketed span and flattens the document's line structure. Plain + * text routinely contains angle brackets that are not markup: an email address + * (`Reply from John `), a markdown autolink + * (``), or a placeholder (``). + */ +const HTML_TAG_PATTERN = + /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)\b[^>]*>/i + +/** + * Reports whether a value carries real HTML markup and is therefore worth routing + * through {@link htmlToPlainText}. Use this instead of a hand-rolled tag test so + * connectors cannot drift apart on what counts as markup. + */ +export function looksLikeHtml(value: string): boolean { + return HTML_TAG_PATTERN.test(value) +} + +/** + * Strips HTML tags from content and decodes HTML entities, including numeric + * character references in decimal (`’`) and hex (`’`) form. + * + * Rendered CMS content — WordPress, Zendesk, Confluence — emits typographic + * punctuation as numeric references, which previously reached the index verbatim. */ export function htmlToPlainText(html: string): string { - let text = html.replace(/<[^>]*>/g, ' ') - text = text - .replace(/ /g, ' ') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/&/g, '&') + const text = html + .replace(/<[^>]*>/g, ' ') + .replace(HTML_ENTITY_PATTERN, (raw: string, hex?: string, decimal?: string, named?: string) => { + if (named !== undefined) return NAMED_ENTITIES[named] ?? raw + if (hex !== undefined) return decodeCharacterReference(raw, Number.parseInt(hex, 16)) + if (decimal !== undefined) return decodeCharacterReference(raw, Number.parseInt(decimal, 10)) + return raw + }) return text.replace(/\s+/g, ' ').trim() } diff --git a/apps/sim/connectors/webflow/webflow.test.ts b/apps/sim/connectors/webflow/webflow.test.ts index 25efd507f67..15468d0babc 100644 --- a/apps/sim/connectors/webflow/webflow.test.ts +++ b/apps/sim/connectors/webflow/webflow.test.ts @@ -1,8 +1,8 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { isCurrentItem } from '@/connectors/webflow/webflow' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { isCurrentItem, webflowConnector } from '@/connectors/webflow/webflow' describe('isCurrentItem', () => { it.concurrent('keeps items explicitly not archived', () => { @@ -47,3 +47,92 @@ describe('isCurrentItem', () => { expect(items.filter(isCurrentItem).map((i) => i.id)).toEqual(['a', 'c', 'd']) }) }) + +const ACCESS_TOKEN = 'test-token' +const CONFIG = { siteId: 'site-1', collectionId: 'col-1' } + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +function itemFixture(id: string) { + return { id, fieldData: { name: id, slug: id }, lastUpdated: '2026-01-01T00:00:00Z' } +} + +/** + * The collection-name lookup fires before the items request, so every listing + * exercise queues that response first. + */ +function mockNameThenItems(itemsBody: unknown) { + mockFetch + .mockResolvedValueOnce(jsonResponse({ id: 'col-1', displayName: 'Posts' })) + .mockResolvedValueOnce(jsonResponse(itemsBody)) +} + +describe('webflow listDocuments deletion-reconciliation guards', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('leaves listingCapped unset when the cap lands exactly on collection exhaustion', async () => { + mockNameThenItems({ items: [itemFixture('a')], pagination: { total: 1 } }) + + const syncContext: Record = {} + await webflowConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('flags listingCapped when the cap stops short of the collection total', async () => { + mockNameThenItems({ items: [itemFixture('a')], pagination: { total: 10 } }) + + const syncContext: Record = {} + await webflowConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + /** + * Without a usable `pagination.total` the offset math cannot tell a full page + * apart from the last one, so treating it as exhausted would feed every unread + * row to deletion reconciliation. + */ + it('flags listingCapped on a full page whose envelope carries no usable total', async () => { + const items = Array.from({ length: 100 }, (_, i) => itemFixture(`item-${i}`)) + mockNameThenItems({ items }) + + const syncContext: Record = {} + await webflowConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset on a short page with no usable total', async () => { + mockNameThenItems({ items: [itemFixture('a')] }) + + const syncContext: Record = {} + await webflowConnector.listDocuments(ACCESS_TOKEN, CONFIG, undefined, syncContext) + + expect(syncContext.listingCapped).toBeUndefined() + }) +}) diff --git a/apps/sim/connectors/webflow/webflow.ts b/apps/sim/connectors/webflow/webflow.ts index 31e3ffd3707..3a716a380bc 100644 --- a/apps/sim/connectors/webflow/webflow.ts +++ b/apps/sim/connectors/webflow/webflow.ts @@ -12,8 +12,20 @@ const PAGE_SIZE = 100 interface WebflowCollection { id: string - displayName: string - slug: string + displayName?: string + slug?: string +} + +/** + * Headers for every Webflow Data API v2 call. The API version is carried by the + * `/v2` path segment — v2 has no `accept-version` header (that was the v1 + * convention), so only bearer auth and a JSON `accept` are sent. + */ +function webflowHeaders(accessToken: string): Record { + return { + Authorization: `Bearer ${accessToken}`, + accept: 'application/json', + } } interface WebflowItem { @@ -44,12 +56,6 @@ export function isCurrentItem(item: { isArchived?: boolean }): boolean { return item.isArchived !== true } -interface WebflowPagination { - total: number - offset: number - limit: number -} - interface CursorState { collectionIndex: number offset: number @@ -117,7 +123,7 @@ export const webflowConnector: ConnectorConfig = { if (cursor) { cursorState = JSON.parse(cursor) as CursorState } else { - const collections = await fetchCollectionIds(accessToken, siteId, collectionIds) + const collections = await fetchCollections(accessToken, siteId, collectionIds, syncContext) cursorState = { collectionIndex: 0, offset: 0, collections } } @@ -125,10 +131,6 @@ export const webflowConnector: ConnectorConfig = { return { documents: [], hasMore: false } } - if (syncContext && !syncContext.collectionNames) { - syncContext.collectionNames = {} - } - const totalDocsFetched = (syncContext?.totalDocsFetched as number) ?? 0 if (maxItems > 0 && totalDocsFetched >= maxItems) { return { documents: [], hasMore: false } @@ -137,11 +139,19 @@ export const webflowConnector: ConnectorConfig = { const currentCollectionId = cursorState.collections[cursorState.collectionIndex] const collectionName = await fetchCollectionName(accessToken, currentCollectionId, syncContext) + /** + * Never request more rows than the remaining `maxItems` budget — the API caps + * `limit` at 100, and rows fetched past the cap would only be discarded, at + * the cost of quota against Webflow's 60 requests/minute floor on Starter and + * Basic plans. The early return above guarantees the remainder is at least 1. + */ + const pageSize = maxItems > 0 ? Math.min(PAGE_SIZE, maxItems - totalDocsFetched) : PAGE_SIZE + const params = new URLSearchParams() - params.append('limit', String(PAGE_SIZE)) + params.append('limit', String(pageSize)) params.append('offset', String(cursorState.offset)) - const url = `${WEBFLOW_API}/collections/${currentCollectionId}/items?${params.toString()}` + const url = `${WEBFLOW_API}/collections/${encodeURIComponent(currentCollectionId)}/items?${params.toString()}` logger.info('Listing Webflow CMS items', { siteId, @@ -151,10 +161,7 @@ export const webflowConnector: ConnectorConfig = { const response = await fetchWithRetry(url, { method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, + headers: webflowHeaders(accessToken), }) if (!response.ok) { @@ -167,50 +174,68 @@ export const webflowConnector: ConnectorConfig = { } const data = (await response.json()) as { - items: WebflowItem[] - pagination: WebflowPagination + items?: WebflowItem[] + pagination?: { total?: number } } + const rawItems = data.items || [] + /** * Archived items are filtered out before mapping so they leave the listing - * and get purged by deletion reconciliation. Pagination stays driven by the - * raw `pagination.limit`/`pagination.total` from the API, never by the - * filtered count, so cursor math is unaffected. + * and get purged by deletion reconciliation. Pagination advances by the raw + * row count, never the filtered count, so cursor math is unaffected. */ - const items = (data.items || []).filter(isCurrentItem) - const pageDocuments: ExternalDocument[] = items.map((item) => - itemToDocument(item, currentCollectionId, collectionName) - ) - - let documents = pageDocuments - if (maxItems > 0) { - const remaining = Math.max(0, maxItems - totalDocsFetched) - if (documents.length > remaining) { - documents = documents.slice(0, remaining) - } - } + const documents: ExternalDocument[] = rawItems + .filter(isCurrentItem) + .map((item) => itemToDocument(item, currentCollectionId, collectionName)) if (syncContext) { syncContext.totalDocsFetched = totalDocsFetched + documents.length } - const { pagination } = data - const hasMoreInCollection = cursorState.offset + pagination.limit < pagination.total + /** + * `pagination.total` is the collection size per the Data API v2 list-items + * reference, read defensively so a malformed envelope cannot turn the offset + * math into `NaN` (which would silently end the collection mid-listing). The + * offset advances by the rows actually returned rather than the echoed + * `pagination.limit`, so a short page can never skip rows. + */ + const reportedTotal = Number(data.pagination?.total) + const totalKnown = Number.isFinite(reportedTotal) + const total = totalKnown ? reportedTotal : rawItems.length + const advance = rawItems.length + + const hasMoreInCollection = advance > 0 && cursorState.offset + advance < total const hasMoreCollections = cursorState.collectionIndex < cursorState.collections.length - 1 const hitMaxItems = maxItems > 0 && totalDocsFetched + documents.length >= maxItems + + /** + * The page came back empty while the collection still reports unread rows — + * the listing is truncated by a transport or shape fault rather than + * exhausted, so reconciliation must not hard-delete the unlisted rows. + */ + const stalledMidCollection = advance === 0 && cursorState.offset < total + /** - * When the cap stops the sync, flag the listing as capped so the sync engine - * skips deletion reconciliation — otherwise still-existing documents that - * were never listed get hard-deleted. "More" means any of: items dropped - * from this page (`pageDocuments.length > documents.length`), more pages in - * this collection, or more collections still to visit. The within-page drop - * is the only signal when a collection fits in a single API response. + * A full page with no usable `pagination.total` to page against. The fallback + * total collapses to the row count, which ends the collection right here, so + * whether rows remain is unknowable — and guessing "exhausted" would hand + * every unread row to deletion reconciliation. + */ + const unknownTotalOnFullPage = !totalKnown && advance >= pageSize + + /** + * A truncated listing must skip deletion reconciliation, or still-existing + * documents that were never listed get hard-deleted. The cap truncates only + * when rows remain beyond it: more pages in this collection, or more + * collections still to visit. The request already clamps `limit` to the + * remaining budget, so the cap never drops rows from within a page. */ - const droppedWithinPage = documents.length < pageDocuments.length if ( syncContext && - hitMaxItems && - (droppedWithinPage || hasMoreInCollection || hasMoreCollections) + (stalledMidCollection || + unknownTotalOnFullPage || + (hitMaxItems && (hasMoreInCollection || hasMoreCollections))) ) { syncContext.listingCapped = true } @@ -221,7 +246,7 @@ export const webflowConnector: ConnectorConfig = { } else if (hasMoreInCollection) { nextCursor = JSON.stringify({ collectionIndex: cursorState.collectionIndex, - offset: cursorState.offset + pagination.limit, + offset: cursorState.offset + advance, collections: cursorState.collections, }) } else if (hasMoreCollections) { @@ -242,7 +267,8 @@ export const webflowConnector: ConnectorConfig = { getDocument: async ( accessToken: string, _sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { const separatorIndex = externalId.indexOf(':') if (separatorIndex === -1) { @@ -253,14 +279,11 @@ export const webflowConnector: ConnectorConfig = { const docCollectionId = externalId.slice(0, separatorIndex) const itemId = externalId.slice(separatorIndex + 1) - const url = `${WEBFLOW_API}/collections/${docCollectionId}/items/${itemId}` + const url = `${WEBFLOW_API}/collections/${encodeURIComponent(docCollectionId)}/items/${encodeURIComponent(itemId)}` const response = await fetchWithRetry(url, { method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, + headers: webflowHeaders(accessToken), }) if (!response.ok) { @@ -276,7 +299,7 @@ export const webflowConnector: ConnectorConfig = { */ if (!isCurrentItem(item)) return null - const collectionName = await fetchCollectionNameDirect(accessToken, docCollectionId) + const collectionName = await fetchCollectionName(accessToken, docCollectionId, syncContext) return itemToDocument(item, docCollectionId, collectionName) }, @@ -297,16 +320,10 @@ export const webflowConnector: ConnectorConfig = { } try { - const siteUrl = `${WEBFLOW_API}/sites/${siteId}` + const siteUrl = `${WEBFLOW_API}/sites/${encodeURIComponent(siteId)}` const siteResponse = await fetchWithRetry( siteUrl, - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, - }, + { method: 'GET', headers: webflowHeaders(accessToken) }, VALIDATE_RETRY_OPTIONS ) @@ -322,16 +339,10 @@ export const webflowConnector: ConnectorConfig = { } for (const collectionId of collectionIds) { - const collectionUrl = `${WEBFLOW_API}/collections/${collectionId}` + const collectionUrl = `${WEBFLOW_API}/collections/${encodeURIComponent(collectionId)}` const collectionResponse = await fetchWithRetry( collectionUrl, - { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, - }, + { method: 'GET', headers: webflowHeaders(accessToken) }, VALIDATE_RETRY_OPTIONS ) @@ -363,7 +374,7 @@ export const webflowConnector: ConnectorConfig = { const lastModified = parseTagDate(metadata.lastModified) if (lastModified) result.lastModified = lastModified - if (typeof metadata.slug === 'string') { + if (typeof metadata.slug === 'string' && metadata.slug.length > 0) { result.slug = metadata.slug } @@ -400,89 +411,88 @@ function itemToDocument( } /** - * Fetches collection IDs for a site. If a specific collectionId is provided, - * returns only that ID. Otherwise fetches all collections from the site. + * Resolves the collection IDs to sync for a site. Explicitly configured IDs are + * an intentional scope filter and are used as-is; otherwise every collection on + * the site is listed via `GET /sites/{site_id}/collections`. + * + * That listing already carries each collection's `displayName`, so it seeds the + * `syncContext` name cache — otherwise every collection would cost an extra + * `GET /collections/{id}` against Webflow's 60 requests/minute floor. */ -async function fetchCollectionIds( +async function fetchCollections( accessToken: string, siteId: string, - collectionIds: string[] + collectionIds: string[], + syncContext?: Record ): Promise { if (collectionIds.length > 0) { return collectionIds } - const url = `${WEBFLOW_API}/sites/${siteId}/collections` + const url = `${WEBFLOW_API}/sites/${encodeURIComponent(siteId)}/collections` const response = await fetchWithRetry(url, { method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, + headers: webflowHeaders(accessToken), }) if (!response.ok) { throw new Error(`Failed to list Webflow collections: ${response.status}`) } - const data = (await response.json()) as { collections: WebflowCollection[] } - return (data.collections || []).map((c) => c.id) -} - -/** - * Fetches a collection's display name, caching in syncContext. - */ -async function fetchCollectionName( - accessToken: string, - collectionId: string, - syncContext?: Record -): Promise { - const names = (syncContext?.collectionNames ?? {}) as Record - if (names[collectionId]) return names[collectionId] - - const name = await fetchCollectionNameDirect(accessToken, collectionId) + const data = (await response.json()) as { collections?: WebflowCollection[] } + const collections = data.collections || [] if (syncContext) { const cached = (syncContext.collectionNames ?? {}) as Record - cached[collectionId] = name + for (const collection of collections) { + cached[collection.id] = collection.displayName || collection.slug || collection.id + } syncContext.collectionNames = cached } - return name + return collections.map((c) => c.id) } /** - * Fetches a collection's display name directly from the API. + * Resolves a collection's display name, memoized in syncContext for the run. + * + * The name is presentational (it prefixes each item's plain text and populates + * the `collectionName` tag), so a lookup failure degrades to the collection id + * rather than aborting the page — an item is never dropped over a missing label. */ -async function fetchCollectionNameDirect( +async function fetchCollectionName( accessToken: string, - collectionId: string + collectionId: string, + syncContext?: Record ): Promise { + const cached = (syncContext?.collectionNames ?? {}) as Record + if (cached[collectionId]) return cached[collectionId] + + let name = collectionId try { - const url = `${WEBFLOW_API}/collections/${collectionId}` + const url = `${WEBFLOW_API}/collections/${encodeURIComponent(collectionId)}` const response = await fetchWithRetry(url, { method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - 'accept-version': '2.0.0', - }, + headers: webflowHeaders(accessToken), }) - if (!response.ok) { - logger.warn('Failed to fetch collection name', { - collectionId, - status: response.status, - }) - return collectionId + if (response.ok) { + const data = (await response.json()) as WebflowCollection + name = data.displayName || data.slug || collectionId + } else { + logger.warn('Failed to fetch collection name', { collectionId, status: response.status }) } - - const data = (await response.json()) as WebflowCollection - return data.displayName || data.slug || collectionId } catch (error) { logger.warn('Error fetching collection name', { collectionId, error: toError(error).message, }) - return collectionId } + + if (syncContext) { + cached[collectionId] = name + syncContext.collectionNames = cached + } + + return name } diff --git a/apps/sim/connectors/wordpress/meta.ts b/apps/sim/connectors/wordpress/meta.ts index d15244231d3..0d4881ced8e 100644 --- a/apps/sim/connectors/wordpress/meta.ts +++ b/apps/sim/connectors/wordpress/meta.ts @@ -7,7 +7,7 @@ export const wordpressConnectorMeta: ConnectorMeta = { id: 'wordpress', name: 'WordPress', description: - 'Sync posts and pages from a WordPress.com site. OAuth tokens expire after ~2 weeks (no refresh token).', + 'Sync published posts and pages from a WordPress.com site. OAuth tokens expire after ~2 weeks (no refresh token).', version: '1.0.0', icon: WordpressIcon, @@ -20,7 +20,7 @@ export const wordpressConnectorMeta: ConnectorMeta = { type: 'short-input', placeholder: 'e.g. mysite.wordpress.com', required: true, - description: 'WordPress site domain', + description: 'WordPress.com site domain or site ID', }, { id: 'postType', diff --git a/apps/sim/connectors/wordpress/wordpress.test.ts b/apps/sim/connectors/wordpress/wordpress.test.ts new file mode 100644 index 00000000000..580354b4e5a --- /dev/null +++ b/apps/sim/connectors/wordpress/wordpress.test.ts @@ -0,0 +1,244 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { wordpressConnector } from '@/connectors/wordpress/wordpress' + +const ACCESS_TOKEN = 'test-token' +const SITE_CONFIG = { siteUrl: 'mysite.wordpress.com' } + +const mockFetch = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +/** Resolves the URL of the nth (0-indexed) fetch the connector performed. */ +function requestUrl(callIndex = 0): URL { + const call = mockFetch.mock.calls[callIndex] + if (!call) throw new Error(`No fetch call at index ${callIndex}`) + return new URL(String(call[0])) +} + +function postFixture(overrides: Record = {}) { + return { + ID: 1, + title: 'Hello', + content: '

Body

', + URL: 'https://mysite.wordpress.com/2026/01/01/hello/', + modified: '2026-01-02T00:00:00+00:00', + type: 'post', + author: { name: 'Ada' }, + categories: { News: { name: 'News' } }, + tags: { Launch: { name: 'Launch' } }, + ...overrides, + } +} + +describe('wordpress listDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('targets the WordPress.com posts endpoint with a documented page size and field list', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 0, posts: [] })) + + await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG) + + const url = requestUrl() + expect(url.origin).toBe('https://public-api.wordpress.com') + expect(url.pathname).toBe('/rest/v1.1/sites/mysite.wordpress.com/posts') + expect(Number(url.searchParams.get('number'))).toBeLessThanOrEqual(100) + expect(url.searchParams.get('type')).toBe('any') + expect(url.searchParams.get('offset')).toBe('0') + expect(url.searchParams.get('fields')).toContain('modified') + }) + + /** + * The API builds `meta.next_page` as `value=&id=` and omits the + * handle entirely when that column is absent from `fields`. Ordering defaults to + * `date`, so dropping `date` from the projection silently disables the cursor and + * falls back to offset paging, which re-numbers whenever a post is published. + */ + it('keeps the default sort column in the projection so page_handle is returned', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 0, posts: [] })) + + await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG) + + expect(requestUrl().searchParams.get('fields')?.split(',')).toContain('date') + }) + + it('normalizes a pasted site URL down to the bare host', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 0, posts: [] })) + + await wordpressConnector.listDocuments(ACCESS_TOKEN, { + siteUrl: ' https://mysite.wordpress.com/blog/?utm=1 ', + }) + + expect(requestUrl().pathname).toBe('/rest/v1.1/sites/mysite.wordpress.com/posts') + }) + + it('maps postType to the documented type parameter', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 0, posts: [] })) + + await wordpressConnector.listDocuments(ACCESS_TOKEN, { ...SITE_CONFIG, postType: 'Pages' }) + + expect(requestUrl().searchParams.get('type')).toBe('page') + }) + + /** + * WordPress.com renders typographic punctuation as numeric character + * references, so decoding them is what keeps `’` out of the index as the + * literal text `’`. The decode lives in the shared `htmlToPlainText`, + * which every HTML-sourced connector routes through. + */ + it('strips rendered HTML and decodes numeric entities in the title', async () => { + mockFetch.mockResolvedValue( + jsonResponse({ + found: 1, + posts: [postFixture({ title: 'Ada’s launch' })], + }) + ) + + const result = await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG) + + expect(result.documents[0].title).toBe('Ada’s launch') + expect(result.documents[0].content).not.toContain('<') + }) + + it('flags the listing capped when maxPosts hides posts that still exist', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 10, posts: [postFixture({ ID: 1 })] })) + + const syncContext: Record = {} + const result = await wordpressConnector.listDocuments( + ACCESS_TOKEN, + { ...SITE_CONFIG, maxPosts: '1' }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + expect(syncContext.listingCapped).toBe(true) + }) + + it('leaves listingCapped unset when the cap lands exactly on source exhaustion', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 1, posts: [postFixture({ ID: 1 })] })) + + const syncContext: Record = {} + await wordpressConnector.listDocuments( + ACCESS_TOKEN, + { ...SITE_CONFIG, maxPosts: '1' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('requests only the remaining posts on the final capped page', async () => { + mockFetch.mockResolvedValue(jsonResponse({ found: 500, posts: [] })) + + await wordpressConnector.listDocuments( + ACCESS_TOKEN, + { ...SITE_CONFIG, maxPosts: '105' }, + undefined, + { + totalDocsFetched: 100, + } + ) + + expect(requestUrl().searchParams.get('number')).toBe('5') + }) + + it('carries the page_handle cursor forward when the response provides one', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ + found: 10, + posts: [postFixture({ ID: 1 })], + meta: { next_page: 'handle-2' }, + }) + ) + + const first = await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG, undefined, {}) + expect(first.hasMore).toBe(true) + + mockFetch.mockResolvedValueOnce(jsonResponse({ found: 10, posts: [] })) + await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG, first.nextCursor, {}) + + expect(requestUrl(1).searchParams.get('page_handle')).toBe('handle-2') + }) + + it('produces the same contentHash from listDocuments and getDocument', async () => { + mockFetch.mockResolvedValueOnce(jsonResponse({ found: 1, posts: [postFixture()] })) + const listed = await wordpressConnector.listDocuments(ACCESS_TOKEN, SITE_CONFIG) + + mockFetch.mockResolvedValueOnce(jsonResponse(postFixture())) + const fetched = await wordpressConnector.getDocument(ACCESS_TOKEN, SITE_CONFIG, '1') + + expect(fetched?.contentHash).toBe(listed.documents[0].contentHash) + }) +}) + +describe('wordpress getDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('returns null for a missing post', async () => { + mockFetch.mockResolvedValue(jsonResponse({ error: 'unknown_post' }, 404)) + + await expect( + wordpressConnector.getDocument(ACCESS_TOKEN, SITE_CONFIG, '99') + ).resolves.toBeNull() + }) +}) + +describe('wordpress validateConfig', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('rejects a missing site URL without calling the API', async () => { + await expect(wordpressConnector.validateConfig(ACCESS_TOKEN, {})).resolves.toEqual({ + valid: false, + error: 'Site URL is required', + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('rejects a non-positive maxPosts', async () => { + const result = await wordpressConnector.validateConfig(ACCESS_TOKEN, { + ...SITE_CONFIG, + maxPosts: '0', + }) + + expect(result.valid).toBe(false) + }) + + it('accepts a reachable site', async () => { + mockFetch.mockResolvedValue(jsonResponse({ ID: 123, name: 'My Site' })) + + await expect(wordpressConnector.validateConfig(ACCESS_TOKEN, SITE_CONFIG)).resolves.toEqual({ + valid: true, + }) + }) +}) diff --git a/apps/sim/connectors/wordpress/wordpress.ts b/apps/sim/connectors/wordpress/wordpress.ts index f6af045a2c0..90747709cfc 100644 --- a/apps/sim/connectors/wordpress/wordpress.ts +++ b/apps/sim/connectors/wordpress/wordpress.ts @@ -1,8 +1,14 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' +import { + CONNECTOR_MAX_FILE_BYTES, + htmlToPlainText, + joinTagArray, + parseTagDate, + stubOrSkipBySize, +} from '@/connectors/utils' import { DEFAULT_MAX_POSTS, wordpressConnectorMeta } from '@/connectors/wordpress/meta' const logger = createLogger('WordPressConnector') @@ -10,14 +16,33 @@ const logger = createLogger('WordPressConnector') const WP_API_BASE = 'https://public-api.wordpress.com/rest/v1.1/sites' /** - * Strips protocol prefix and trailing slashes from a site URL so the - * WordPress.com API receives a bare domain (e.g. "mysite.wordpress.com"). + * Reduces a user-supplied site URL to the bare `$site` value the WordPress.com + * REST API expects (a domain such as "mysite.wordpress.com", or a numeric site + * ID). Drops the protocol, any userinfo, and any path/query/fragment — the API + * takes the site as a single path segment, so "https://mysite.com/blog/" must + * become "mysite.com". */ function normalizeSiteUrl(raw: string): string { - return raw.replace(/^https?:\/\//, '').replace(/\/+$/, '') + return raw + .trim() + .replace(/^https?:\/\//i, '') + .replace(/^[^/@]*@/, '') + .replace(/[/?#].*$/, '') } -const POSTS_PER_PAGE = 20 +/** WordPress.com caps `number` at 100 per request. */ +const POSTS_PER_PAGE = 100 + +/** + * Post fields the connector actually reads, plus `date`. + * + * `date` is requested even though nothing reads it: the API builds `meta.next_page` + * as `value=&id=`, so it omits the handle entirely unless the + * column it orders by is inside the projection. Ordering defaults to `date`, so + * dropping it silently degrades every sync to offset paging — which re-numbers + * mid-run whenever a post is published, skipping posts. + */ +const POST_FIELDS = 'ID,title,content,URL,modified,type,author,categories,tags,date' interface WordPressPost { ID: number @@ -36,10 +61,14 @@ interface WordPressPost { interface WordPressPostsResponse { found: number posts: WordPressPost[] + meta?: { + next_page?: string + } } interface ListCursor { offset: number + pageHandle?: string } /** @@ -60,15 +89,21 @@ function extractTagNames(tags: Record): string[] { * Converts a WordPress post to an ExternalDocument. */ function postToDocument(post: WordPressPost): ExternalDocument { - const plainText = htmlToPlainText(post.content) - const fullContent = `# ${post.title}\n\n${plainText}` + /** + * WordPress.com returns `title` and `content` as rendered HTML, so both go + * through `htmlToPlainText` — a title carrying entities (`’`) or inline + * markup would otherwise be indexed and displayed raw. + */ + const title = htmlToPlainText(post.title ?? '') + const plainText = htmlToPlainText(post.content ?? '') + const fullContent = `# ${title}\n\n${plainText}` const contentHash = `wordpress:${post.ID}:${post.modified || ''}` - const categories = extractCategoryNames(post.categories) - const tags = extractTagNames(post.tags) + const categories = extractCategoryNames(post.categories ?? {}) + const tags = extractTagNames(post.tags ?? {}) - return { + const document: ExternalDocument = { externalId: String(post.ID), - title: post.title || 'Untitled', + title: title || 'Untitled', content: fullContent, mimeType: 'text/plain', sourceUrl: post.URL, @@ -81,6 +116,12 @@ function postToDocument(post: WordPressPost): ExternalDocument { tags, }, } + + return stubOrSkipBySize( + document, + Buffer.byteLength(fullContent, 'utf8'), + CONNECTOR_MAX_FILE_BYTES + ) } /** @@ -112,19 +153,27 @@ export const wordpressConnector: ConnectorConfig = { } const siteUrl = normalizeSiteUrl(rawSiteUrl) - const maxPosts = sourceConfig.maxPosts ? Number(sourceConfig.maxPosts) : DEFAULT_MAX_POSTS + const parsedMax = Number(sourceConfig.maxPosts) + const maxPosts = Number.isFinite(parsedMax) && parsedMax > 0 ? parsedMax : DEFAULT_MAX_POSTS const type = resolvePostType(sourceConfig.postType as string | undefined) const parsed: ListCursor = cursor ? JSON.parse(cursor) : { offset: 0 } const totalDocsFetched = (syncContext?.totalDocsFetched as number) ?? 0 - const remaining = maxPosts > 0 ? maxPosts - totalDocsFetched : POSTS_PER_PAGE + const remaining = maxPosts - totalDocsFetched if (remaining <= 0) { return { documents: [], hasMore: false } } const pageSize = Math.min(POSTS_PER_PAGE, remaining) - const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts?number=${pageSize}&offset=${parsed.offset}&type=${type}` + /** + * `page_handle` is the API's documented (and cheapest) cursor; `offset` is the + * fallback for the first page and for any response that omits `meta.next_page`. + */ + const pageParam = parsed.pageHandle + ? `page_handle=${encodeURIComponent(parsed.pageHandle)}` + : `offset=${parsed.offset}` + const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts?number=${pageSize}&${pageParam}&type=${type}&fields=${POST_FIELDS}` logger.info('Fetching WordPress posts', { siteUrl, offset: parsed.offset, type, pageSize }) @@ -148,15 +197,42 @@ export const wordpressConnector: ConnectorConfig = { const totalFetched = totalDocsFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitLimit = maxPosts > 0 && totalFetched >= maxPosts + const hitLimit = totalFetched >= maxPosts const newOffset = parsed.offset + posts.length - const hasMore = !hitLimit && newOffset < data.found + const moreAvailable = posts.length > 0 && newOffset < data.found + const hasMore = !hitLimit && moreAvailable + + /** + * An empty page while the site still reports unread posts is a truncated + * listing, not an exhausted one, so paging stops without having seen every + * post. + */ + const stalledMidListing = posts.length === 0 && parsed.offset < data.found + + /** + * A truncated listing must suppress deletion reconciliation — the sync engine + * hard-deletes every stored document absent from a full listing, and the + * unlisted posts still exist on the site. Set only when posts genuinely + * remain: a cap that lands exactly on exhaustion is a complete listing, and + * flagging it would block deletion reconciliation forever. + */ + if (syncContext && (stalledMidListing || (hitLimit && moreAvailable))) { + syncContext.listingCapped = true + logger.info('WordPress post listing truncated', { + siteUrl, + maxPosts, + found: data.found, + stalledMidListing, + }) + } + + const nextCursor: ListCursor = { offset: newOffset, pageHandle: data.meta?.next_page } return { documents, hasMore, - nextCursor: hasMore ? JSON.stringify({ offset: newOffset }) : undefined, + nextCursor: hasMore ? JSON.stringify(nextCursor) : undefined, } }, @@ -171,31 +247,28 @@ export const wordpressConnector: ConnectorConfig = { } const siteUrl = normalizeSiteUrl(rawSiteUrl) - const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts/${externalId}` + const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts/${encodeURIComponent(externalId)}?fields=${POST_FIELDS}` - try { - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - if (response.status === 404) return null - throw new Error(`WordPress API error: ${response.status}`) - } + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) - const post = (await response.json()) as WordPressPost - return postToDocument(post) - } catch (error) { - logger.warn('Failed to get WordPress document', { - externalId, - error: toError(error).message, - }) - return null + /** + * Only a deleted post (404) resolves to `null`. Every other failure throws so + * the sync engine records a visible failed document instead of dropping the + * post from the run with no counter and no error log. + */ + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`WordPress API error: ${response.status}`) } + + const post = (await response.json()) as WordPressPost + return postToDocument(post) }, validateConfig: async ( @@ -209,8 +282,11 @@ export const wordpressConnector: ConnectorConfig = { return { valid: false, error: 'Site URL is required' } } const siteUrl = normalizeSiteUrl(rawSiteUrl) + if (!siteUrl) { + return { valid: false, error: 'Site URL is required' } + } - if (maxPosts && (Number.isNaN(Number(maxPosts)) || Number(maxPosts) <= 0)) { + if (maxPosts && (!Number.isFinite(Number(maxPosts)) || Number(maxPosts) <= 0)) { return { valid: false, error: 'Max posts must be a positive number' } } @@ -232,6 +308,12 @@ export const wordpressConnector: ConnectorConfig = { if (response.status === 404) { return { valid: false, error: `Site not found: ${siteUrl}` } } + if (response.status === 401 || response.status === 403) { + return { + valid: false, + error: `WordPress authorization failed (${response.status}) — reconnect the account, or check that it can access ${siteUrl}`, + } + } return { valid: false, error: `WordPress API error: ${response.status}` } } diff --git a/apps/sim/connectors/x/meta.ts b/apps/sim/connectors/x/meta.ts index ba82605d4f7..b68af637a94 100644 --- a/apps/sim/connectors/x/meta.ts +++ b/apps/sim/connectors/x/meta.ts @@ -50,7 +50,8 @@ export const xConnectorMeta: ConnectorMeta = { { label: 'Exclude replies', id: 'false' }, { label: 'Include replies', id: 'true' }, ], - description: 'Whether to include reply posts. Applies to "My posts" and "Another user".', + description: + 'Whether to include reply posts. Applies to "My posts" and "Another user". Excluding replies also narrows how far back X will serve a timeline — 800 posts instead of 3,200.', }, { id: 'includeRetweets', @@ -90,7 +91,7 @@ export const xConnectorMeta: ConnectorMeta = { required: false, placeholder: `e.g. 100 (default: ${DEFAULT_MAX_POSTS})`, description: - 'Maximum number of posts to sync (across all configured users). Posts beyond this limit are not deleted from the knowledge base; X also only exposes a limited recent window (≈3,200 timeline posts, ≈800 bookmarks), so posts that age out of that window are removed on the next sync.', + 'Maximum number of posts to sync. The limit is shared across all configured users and spent in the order they are listed, so the last username is the one truncated. Posts beyond this limit are not deleted from the knowledge base; X itself only exposes a limited recent window (up to 3,200 timeline posts and 800 mentions), so posts that age out of that window are removed on the next sync.', }, ], diff --git a/apps/sim/connectors/x/x.ts b/apps/sim/connectors/x/x.ts index 814893928f8..b94aaefb13d 100644 --- a/apps/sim/connectors/x/x.ts +++ b/apps/sim/connectors/x/x.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -17,10 +17,16 @@ const POSTS_PER_PAGE = 100 */ const MIN_PAGE_SIZE = 5 /** - * `edit_history_tweet_ids` is requested explicitly (it is not a default field) so the - * content hash can key on edit-history length and detect edits. + * Requestable post fields. Every value here must appear in the API's field enum — + * X rejects the whole request with a 400 for an unrecognized one. + * + * The edit-history array is deliberately absent: it is NOT an accepted field + * value (the enum carries `edit_controls`, never the history array), and it does + * not need to be requested because it is one of the three fields a post lookup + * returns by default alongside `id` and `text`. `tweetContentHash` therefore + * reads it straight off the response. */ -const TWEET_FIELDS = 'created_at,public_metrics,text,edit_history_tweet_ids' +const TWEET_FIELDS = 'created_at,public_metrics,text' /** * Sync mode determines which timeline the connector reads. @@ -54,7 +60,16 @@ interface XTweet { created_at?: string author_id?: string public_metrics?: XPublicMetrics + /** + * Default-returned edit-history chain, under two names X has not reconciled: + * the prose docs and data dictionary document `edit_history_tweet_ids`, while + * the current OpenAPI `Post` schema declares only `edit_history_post_ids`. + * Both are read because the two official sources disagree about which one the + * wire carries, and reading only one would silently degrade the content hash + * to its `created_at` fallback. + */ edit_history_tweet_ids?: string[] + edit_history_post_ids?: string[] } interface XUser { @@ -63,17 +78,31 @@ interface XUser { username?: string } +/** + * A single entry of the X API `errors[]` array (a "Problem" object). X returns + * these alongside a 200 `data` payload for partial failures, so they must be + * inspected rather than assumed to mean the whole request failed. + */ +interface XProblem { + detail?: string + title?: string + type?: string + parameter?: string + value?: string + resource_type?: string +} + interface XListResponse { data?: XTweet[] includes?: { users?: XUser[] } meta?: { next_token?: string; result_count?: number } - errors?: Array<{ detail?: string; title?: string }> + errors?: XProblem[] } interface XSingleResponse { data?: XTweet includes?: { users?: XUser[] } - errors?: Array<{ detail?: string; title?: string }> + errors?: XProblem[] } /** @@ -128,6 +157,23 @@ function readTrimmed(value: unknown): string | undefined { return trimmed.length > 0 ? trimmed : undefined } +/** + * Normalizes a user-entered timestamp to the second-precision UTC form the X API + * documents for `start_time` / `end_time` (`YYYY-MM-DDTHH:mm:ssZ`). + * + * Users routinely type a bare date (`2024-01-01`) or a local-offset timestamp, + * both of which `Date` parses but the X API rejects. Milliseconds are stripped + * because the documented format carries none. Returns undefined for blank or + * unparseable input so the parameter is simply omitted rather than sent invalid. + */ +function toXTimestamp(value: unknown): string | undefined { + const raw = readTrimmed(value) + if (!raw) return undefined + const parsed = new Date(raw) + if (Number.isNaN(parsed.getTime())) return undefined + return `${parsed.toISOString().slice(0, 19)}Z` +} + /** * Performs an authenticated GET against the X API v2 and returns the parsed JSON. */ @@ -202,14 +248,18 @@ async function resolveUsernameId( * Builds a deterministic, metadata-based content hash for a tweet. * * Tweets are immutable outside the brief post-publish edit window; an edit - * appends a new ID to `edit_history_tweet_ids`. We therefore key the hash on - * the edit-history length when present (so edits are detected as changes), and - * fall back to `created_at` when the field is absent. + * appends a new ID to the edit-history chain. We therefore key the hash on the + * edit-history length when present (so edits are detected as changes), and fall + * back to `created_at` when the field is absent. Both the legacy and current + * names for the chain are accepted so the hash does not shift if X switches the + * wire field name. + * + * Both `listDocuments` and `getDocument` route through this function and request + * the same fields, so a post's hash is identical whichever produced it. */ function tweetContentHash(tweet: XTweet): string { - const historyLength = Array.isArray(tweet.edit_history_tweet_ids) - ? tweet.edit_history_tweet_ids.length - : undefined + const history = tweet.edit_history_tweet_ids ?? tweet.edit_history_post_ids + const historyLength = Array.isArray(history) ? history.length : undefined const changeIndicator = historyLength ?? tweet.created_at ?? '' return `x:${tweet.id}:${changeIndicator}` } @@ -261,6 +311,35 @@ function tweetToDocument(tweet: XTweet, author?: XUser): ExternalDocument { } } +/** + * The one `resource_type` known NOT to cost us a post: a problem about the + * `user` resource means only that the `author_id` expansion could not be + * hydrated, and the post itself is still present in `data`. + */ +const AUTHOR_PROBLEM_RESOURCE_TYPE = 'user' + +/** + * Counts partial-failure entries that may describe a post X refused to return. + * + * X answers a listing with HTTP 200 and a populated `data` array while still + * reporting per-resource problems in `errors[]` (not-found, unavailable, or + * not-authorized resources). Everything that is not an author-expansion problem + * is counted, rather than matching one post-side `resource_type` literal: X's + * OpenAPI declared a closed enum for that field through spec 2.61 and had + * dropped it by 2.167, so the post-side spelling is no longer constrained by the + * contract and may follow the Tweet→Post rename at any time. Matching a literal + * would then silently start counting zero — the exact failure this guards. + * + * The asymmetry is deliberate: over-counting suppresses deletion reconciliation + * for one sync (and a forced full sync overrides it), while under-counting lets + * the engine hard-delete a document for a post that still exists and merely + * could not be served on this page. + */ +function omittedPostCount(errors: XProblem[] | undefined): number { + if (!errors?.length) return 0 + return errors.filter((problem) => problem.resource_type !== AUTHOR_PROBLEM_RESOURCE_TYPE).length +} + /** * Maps tweets from a list response to documents, joining each tweet to its * author via the `includes.users` expansion (matched on `author_id`). @@ -319,8 +398,8 @@ function buildListParams( } if (DATE_RANGE_CAPABLE_MODES.has(mode)) { - const startTime = readTrimmed(sourceConfig.startTime) - const endTime = readTrimmed(sourceConfig.endTime) + const startTime = toXTimestamp(sourceConfig.startTime) + const endTime = toXTimestamp(sourceConfig.endTime) if (startTime) params.start_time = startTime if (endTime) params.end_time = endTime } @@ -358,9 +437,11 @@ export const xConnector: ConnectorConfig = { return { documents: [], hasMore: false } } - // For the multi-username "user" mode, walk one username per cursor cycle. The - // cursor packs the username index and that user's pagination token; the shared - // cap is enforced across all users via syncContext.collected. + /** + * For the multi-username "user" mode, walk one username per cursor cycle. + * The cursor packs the username index and that user's pagination token; the + * shared cap is enforced across all users via `syncContext.collected`. + */ const usernames = mode === 'user' ? parseUsernames(sourceConfig.username) : [] if (mode === 'user' && usernames.length === 0) { throw new Error('Username is required when Sync Mode is "Another user"') @@ -377,11 +458,23 @@ export const xConnector: ConnectorConfig = { } } - // Resolve the target user ID. For `user` mode it depends on the current index - // (resolved per page, cheap); for self-modes it is cached on syncContext. + /** + * Resolve the target user ID. Both branches cache on syncContext so a + * multi-page run performs at most one lookup per distinct account. + */ let userId: string if (mode === 'user') { - userId = await resolveUsernameId(accessToken, usernames[userIndex]) + /** + * Cache handle → id for the whole run. X's user-lookup endpoint has a far + * tighter rate limit than the timeline endpoints, and without this cache a + * multi-page sync spends one lookup per page on the same handle. + */ + const handle = usernames[userIndex] + const idsByHandle = (syncContext?.userIdsByHandle as Record | undefined) ?? {} + userId = idsByHandle[handle] ?? (await resolveUsernameId(accessToken, handle)) + if (syncContext) { + syncContext.userIdsByHandle = { ...idsByHandle, [handle]: userId } + } } else { userId = (syncContext?.userId as string | undefined) ?? (await resolveMyUserId(accessToken)) if (syncContext) syncContext.userId = userId @@ -399,9 +492,28 @@ export const xConnector: ConnectorConfig = { throw new Error(response.errors[0]?.detail || response.errors[0]?.title || 'X API error') } + /** + * X reports per-resource failures in `errors[]` on an otherwise successful + * 200. A `tweet` problem means a still-existing post was withheld from this + * page, so the listing no longer represents the full source and deletion + * reconciliation would hard-delete a document that is merely unavailable + * right now. + */ + const omittedPosts = omittedPostCount(response.errors) + if (response.errors?.length) { + logger.warn('X returned partial errors alongside listing data', { + mode, + userId, + omittedPosts, + errors: response.errors.map((problem) => problem.title ?? problem.detail ?? problem.type), + }) + } + if (omittedPosts > 0 && syncContext) syncContext.listingCapped = true + let documents = mapTweets(response) - if (maxPosts > 0 && collectedSoFar + documents.length > maxPosts) { + const slicedByCap = maxPosts > 0 && collectedSoFar + documents.length > maxPosts + if (slicedByCap) { documents = documents.slice(0, maxPosts - collectedSoFar) } const newCollected = collectedSoFar + documents.length @@ -409,16 +521,23 @@ export const xConnector: ConnectorConfig = { const capReached = maxPosts > 0 && newCollected >= maxPosts const nextToken = response.meta?.next_token + const moreUsernames = mode === 'user' && userIndex + 1 < usernames.length - // Advance pagination: continue the current user's pages, else move to the next - // username (user mode), else stop. + /** + * Advance pagination: continue the current user's pages, else move to the + * next username (user mode), else stop. + */ if (capReached) { - // We stopped before exhausting the source, so the listing is incomplete: - // older previously-synced posts may still exist beyond the `maxPosts` cap. - // Flag the sync as capped so the engine skips deletion reconciliation and - // does not soft-delete posts that simply fell outside this run's window. - // A forced full sync bypasses this guard and reconciles normally. - if (syncContext) syncContext.listingCapped = true + /** + * Only flag the run as capped when posts actually remain unlisted — a page + * trimmed by the cap, a further page token, or another configured account. + * When the cap merely coincides with source exhaustion the listing IS + * complete, and flagging it would permanently suppress deletion + * reconciliation so posts removed at the source were never cleaned up. + */ + if (syncContext && (slicedByCap || nextToken || moreUsernames)) { + syncContext.listingCapped = true + } return { documents, hasMore: false } } @@ -445,25 +564,23 @@ export const xConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - const response = (await xApiGet(`/tweets/${encodeURIComponent(externalId)}`, accessToken, { - 'tweet.fields': TWEET_FIELDS, - expansions: 'author_id', - 'user.fields': 'name,username', - })) as XSingleResponse - - const tweet = response.data - if (!tweet) return null - - const author = response.includes?.users?.find((u) => u.id === tweet.author_id) - return tweetToDocument(tweet, author) - } catch (error) { - logger.warn('Failed to get X tweet document', { - externalId, - error: toError(error).message, - }) - return null - } + const response = (await xApiGet(`/tweets/${encodeURIComponent(externalId)}`, accessToken, { + 'tweet.fields': TWEET_FIELDS, + expansions: 'author_id', + 'user.fields': 'name,username', + })) as XSingleResponse + + /** + * X answers a deleted or newly-protected post with HTTP 200, an absent `data`, + * and an `errors` entry — that is the only absence. Transport, auth, and + * rate-limit failures throw out of `xApiGet` so the sync engine records a + * visible failed document instead of dropping the post with no counter. + */ + const tweet = response.data + if (!tweet) return null + + const author = response.includes?.users?.find((u) => u.id === tweet.author_id) + return tweetToDocument(tweet, author) }, validateConfig: async ( diff --git a/apps/sim/connectors/youtube/youtube.test.ts b/apps/sim/connectors/youtube/youtube.test.ts index 966156fc041..2eb8fe7242d 100644 --- a/apps/sim/connectors/youtube/youtube.test.ts +++ b/apps/sim/connectors/youtube/youtube.test.ts @@ -212,8 +212,16 @@ describe('youtubeConnector.listDocuments', () => { excludeShorts: 'true', }) - expect(result.documents.map((d) => d.externalId)).toEqual(['aaa']) + // 'bbb' is a real Short and is intentionally out of scope, so it leaves the listing + // and reconciles normally. 'ccc' is merely absent from videos.list — absence is not + // deletion, so it stays in the listing as a deferred stub for getDocument to judge. + expect(result.documents.map((d) => d.externalId)).toEqual(['aaa', 'ccc']) expect(result.documents[0].contentDeferred).toBe(false) + expect(result.documents[1]).toMatchObject({ + contentDeferred: true, + content: '', + contentHash: 'youtube:ccc:2024-01-01T00:00:00Z', + }) expect(urls[1]).toContain('/videos?part=snippet%2CcontentDetails%2Cstatus') }) @@ -271,6 +279,120 @@ describe('youtubeConnector.listDocuments', () => { listDocuments(API_KEY, { playlistId: PLAYLIST_ID, excludeShorts: 'true' }) ).rejects.toThrow('Failed to batch-fetch YouTube videos: 403') }) + + it('caps the listing when maxVideos trims videos off the last page', async () => { + mockFetch(playlistOnly([item('aaa'), item('bbb'), item('ccc')])) + + const syncContext: Record = {} + const result = await listDocuments( + API_KEY, + { playlistId: PLAYLIST_ID, maxVideos: '2' }, + undefined, + syncContext + ) + + // 'ccc' exists at the source but was trimmed by the cap, so reconciling deletions + // against this listing would hard-delete it. + expect(result.documents.map((d) => d.externalId)).toEqual(['aaa', 'bbb']) + expect(syncContext.listingCapped).toBe(true) + expect(result.hasMore).toBe(false) + }) + + it('does not cap when maxVideos coincides exactly with an exhausted source', async () => { + mockFetch(playlistOnly([item('aaa'), item('bbb')])) + + const syncContext: Record = {} + const result = await listDocuments( + API_KEY, + { playlistId: PLAYLIST_ID, maxVideos: '2' }, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['aaa', 'bbb']) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('keeps paginating past an out-of-cutoff item instead of breaking early', async () => { + const dated = (videoId: string, videoPublishedAt: string): TestPlaylistItem => ({ + contentDetails: { videoId, videoPublishedAt }, + snippet: { title: 'A video' }, + }) + + mockFetch( + playlistOnly( + [dated('old', '2020-01-01T00:00:00Z'), dated('new', '2024-06-01T00:00:00Z')], + 'TOKEN2' + ) + ) + + const result = await listDocuments(API_KEY, { + channelId: 'UC_x5XG1OV2P6uZZ5FSM9Ttw', + playlistId: PLAYLIST_ID, + publishedAfter: '2024-01-01', + }) + + // Playlist ordering is undocumented, so an old item must not stop the scan — the + // newer item after it still has to reach the listing. + expect(result.documents.map((d) => d.externalId)).toEqual(['new']) + expect(result.hasMore).toBe(true) + expect(result.nextCursor).toBe('TOKEN2') + }) + + it('resolves a bare channel reference as a handle before falling back to forUsername', async () => { + const urls = mockFetch((url) => { + if (url.includes('/channels')) { + return fakeResponse({ + body: url.includes('forHandle') + ? { items: [{ contentDetails: { relatedPlaylists: { uploads: 'UU123' } } }] } + : { items: [] }, + }) + } + return fakeResponse({ body: playlistPage([item('aaa', 'public')]) }) + }) + + const result = await listDocuments(API_KEY, { channelId: 'mkbhd' }) + + expect(urls[0]).toContain('forHandle=mkbhd') + expect(urls.some((u) => u.includes('forUsername'))).toBe(false) + expect(urls[1]).toContain('playlistId=UU123') + expect(result.documents.map((d) => d.externalId)).toEqual(['aaa']) + }) + + it('falls back to forUsername when a bare reference is not a handle', async () => { + const urls = mockFetch((url) => { + if (url.includes('/channels')) { + return fakeResponse({ + body: url.includes('forUsername') + ? { items: [{ contentDetails: { relatedPlaylists: { uploads: 'UU999' } } }] } + : { items: [] }, + }) + } + return fakeResponse({ body: playlistPage([item('aaa', 'public')]) }) + }) + + await listDocuments(API_KEY, { channelId: 'LegacyUser' }) + + expect(urls[0]).toContain('forHandle=LegacyUser') + expect(urls[1]).toContain('forUsername=LegacyUser') + expect(urls[2]).toContain('playlistId=UU999') + }) + + it('resolves a UC channel id with the id filter and never tries a handle', async () => { + const channelId = `UC${'a'.repeat(22)}` + const urls = mockFetch((url) => + url.includes('/channels') + ? fakeResponse({ + body: { items: [{ contentDetails: { relatedPlaylists: { uploads: 'UUabc' } } }] }, + }) + : fakeResponse({ body: playlistPage([item('aaa', 'public')]) }) + ) + + await listDocuments(API_KEY, { channelId }) + + expect(urls[0]).toContain(`id=${channelId}`) + expect(urls.some((u) => u.includes('forHandle') || u.includes('forUsername'))).toBe(false) + }) }) describe('youtubeConnector.getDocument', () => { @@ -302,16 +424,17 @@ describe('youtubeConnector.getDocument', () => { expect(await getDocument(API_KEY, {}, 'aaa')).toBe(null) }) - it('returns null on 403 and 404 without throwing', async () => { - mockFetch(() => fakeResponse({ status: 403, text: 'forbidden' })) - expect(await getDocument(API_KEY, {}, 'aaa')).toBe(null) - - vi.unstubAllGlobals() + it('returns null on 404 without throwing', async () => { mockFetch(() => fakeResponse({ status: 404, text: 'notFound' })) expect(await getDocument(API_KEY, {}, 'aaa')).toBe(null) }) - it('swallows transport failures and returns null', async () => { + it('throws on 403 so quota exhaustion is not mistaken for a deleted video', async () => { + mockFetch(() => fakeResponse({ status: 403, text: 'quotaExceeded' })) + await expect(getDocument(API_KEY, {}, 'aaa')).rejects.toThrow('403') + }) + + it('rethrows transport failures so the sync engine records them', async () => { vi.stubGlobal( 'fetch', vi.fn(async () => { @@ -319,6 +442,6 @@ describe('youtubeConnector.getDocument', () => { }) ) - expect(await getDocument(API_KEY, {}, 'aaa')).toBe(null) + await expect(getDocument(API_KEY, {}, 'aaa')).rejects.toThrow('boom') }) }) diff --git a/apps/sim/connectors/youtube/youtube.ts b/apps/sim/connectors/youtube/youtube.ts index b80ed9117a2..4be5cbfa63a 100644 --- a/apps/sim/connectors/youtube/youtube.ts +++ b/apps/sim/connectors/youtube/youtube.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' @@ -98,28 +98,18 @@ function parseIso8601Duration(value: string | undefined): number | null { } /** - * Resolves a channel reference to its "uploads" playlist ID via `channels.list`. - * - * Accepts a `UC…` channel ID, an `@handle` (resolved with `forHandle`), or a legacy - * username (resolved with `forUsername`). Returns null when the channel is missing or - * has no uploads playlist. + * Issues a single `channels.list` call filtered by exactly one of `id`, `forHandle`, or + * `forUsername`, and returns the first channel's uploads playlist ID (null when the + * filter matched nothing). A `list` call costs a flat 1 quota unit. */ -async function resolveUploadsPlaylistId( +async function lookupUploadsPlaylist( apiKey: string, - channelRef: string, + filter: 'id' | 'forHandle' | 'forUsername', + value: string, retryOptions?: Parameters[2] ): Promise { - const ref = channelRef.trim() - if (!ref) return null - const params = new URLSearchParams({ part: 'contentDetails', key: apiKey }) - if (ref.startsWith('@')) { - params.set('forHandle', ref) - } else if (/^UC[\w-]{20,}$/.test(ref)) { - params.set('id', ref) - } else { - params.set('forUsername', ref) - } + params.set(filter, value) const url = `${YOUTUBE_API_BASE}/channels?${params.toString()}` @@ -132,7 +122,7 @@ async function resolveUploadsPlaylistId( if (!response.ok) { const errorText = await response.text().catch(() => '') logger.error('Failed to resolve channel uploads playlist', { - channelRef: ref, + filter, status: response.status, error: errorText.slice(0, 500), }) @@ -147,25 +137,56 @@ async function resolveUploadsPlaylistId( } /** - * Resolves the effective playlist ID to sync from sourceConfig, and whether the source - * is a channel's reverse-chronological uploads playlist (which enables early-stop for - * the `publishedAfter` filter). A `playlistId` takes precedence over a `channelId`. + * Resolves a channel reference to its "uploads" playlist ID via `channels.list`. + * + * A value shaped like a channel ID is looked up with `id` first. That shape is only a + * heuristic for ordering the attempts, never a rejection: Google does not document a + * channel-ID format at all — the `UC` prefix is an observation from the example IDs in + * the channel-ID guide, not a documented guarantee — so the test stays deliberately + * loose and a miss simply falls through. + * + * Anything else — and a `UC…`-shaped value that matched no channel — is tried as a + * handle: `channels.list` documents that the `forHandle` value "can be prepended with an + * `@` symbol", i.e. the `@` is optional, so a bare `mkbhd` is a valid handle. Only a + * non-`@` reference that is not a handle either falls through to the legacy `forUsername` + * filter. Each fallback costs one extra quota unit and only runs after a miss. Returns + * null when no filter matches a channel with an uploads playlist. + */ +async function resolveUploadsPlaylistId( + apiKey: string, + channelRef: string, + retryOptions?: Parameters[2] +): Promise { + const ref = channelRef.trim() + if (!ref) return null + + if (/^UC[\w-]{20,}$/.test(ref)) { + const byId = await lookupUploadsPlaylist(apiKey, 'id', ref, retryOptions) + if (byId) return byId + } + + const byHandle = await lookupUploadsPlaylist(apiKey, 'forHandle', ref, retryOptions) + if (byHandle || ref.startsWith('@')) return byHandle + + return lookupUploadsPlaylist(apiKey, 'forUsername', ref, retryOptions) +} + +/** + * Resolves the effective playlist ID to sync from sourceConfig. A `playlistId` takes + * precedence over a `channelId`, which resolves to that channel's uploads playlist. */ async function resolvePlaylistId( apiKey: string, sourceConfig: Record, retryOptions?: Parameters[2] -): Promise<{ playlistId: string | null; isUploadsPlaylist: boolean }> { +): Promise { const playlistId = (sourceConfig.playlistId as string | undefined)?.trim() - if (playlistId) return { playlistId, isUploadsPlaylist: false } + if (playlistId) return playlistId const channelId = (sourceConfig.channelId as string | undefined)?.trim() - if (channelId) { - const resolved = await resolveUploadsPlaylistId(apiKey, channelId, retryOptions) - return { playlistId: resolved, isUploadsPlaylist: resolved != null } - } + if (channelId) return resolveUploadsPlaylistId(apiKey, channelId, retryOptions) - return { playlistId: null, isUploadsPlaylist: false } + return null } /** @@ -455,18 +476,11 @@ export const youtubeConnector: ConnectorConfig = { return { documents: [], hasMore: false } } - const cachedPlaylistId = syncContext?.resolvedPlaylistId as string | undefined - let playlistId: string | null = cachedPlaylistId ?? null - let isUploadsPlaylist = (syncContext?.isUploadsPlaylist as boolean | undefined) ?? false + let playlistId = (syncContext?.resolvedPlaylistId as string | undefined) ?? null if (!playlistId) { - const resolved = await resolvePlaylistId(apiKey, sourceConfig) - playlistId = resolved.playlistId - isUploadsPlaylist = resolved.isUploadsPlaylist - if (syncContext) { - if (playlistId) syncContext.resolvedPlaylistId = playlistId - syncContext.isUploadsPlaylist = isUploadsPlaylist - } + playlistId = await resolvePlaylistId(apiKey, sourceConfig) + if (syncContext && playlistId) syncContext.resolvedPlaylistId = playlistId } if (!playlistId) { @@ -515,8 +529,15 @@ export const youtubeConnector: ConnectorConfig = { const excludeShorts = String(sourceConfig.excludeShorts ?? '') === 'true' const keptItems: PlaylistItem[] = [] - let stopEarly = false + /** + * The `publishedAfter` cutoff is applied per item across every page, never as an + * early break. `playlistItems.list` documents no ordering whatsoever — not a + * reverse-chronological guarantee, and not a caveat denying one — so breaking on + * the first out-of-range item would, on any playlist that is not strictly ordered, + * drop every later in-scope video from the listing and hand it straight to + * deletion reconciliation. Draining the cursor costs 1 quota unit per 50 items. + */ for (const item of items) { if (!getVideoId(item)) continue @@ -525,16 +546,7 @@ export const youtubeConnector: ConnectorConfig = { if (publishedAfter != null) { const videoPublishedAt = item.contentDetails?.videoPublishedAt const ms = videoPublishedAt ? new Date(videoPublishedAt).getTime() : Number.NaN - if (!Number.isNaN(ms) && ms < publishedAfter) { - // Uploads playlists are reverse-chronological by publish date, so once we - // cross the cutoff no later item can qualify — stop paginating. For arbitrary - // playlists we only filter per-item (order is not guaranteed). - if (isUploadsPlaylist) { - stopEarly = true - break - } - continue - } + if (!Number.isNaN(ms) && ms < publishedAfter) continue } keptItems.push(item) @@ -568,16 +580,25 @@ export const youtubeConnector: ConnectorConfig = { } } else { for (const item of keptItems) { - /** - * This branch cannot emit a document without the hydrated video, since the - * Shorts decision needs `contentDetails.duration`. An item absent from a trusted - * `videos.list` is skipped because there is nothing to build from — emitting a - * stub instead would re-hydrate to null on every sync. Deletion is never - * inferred from that absence: items whose video is explicitly gone were already - * removed above by `isPlaylistItemPrivate`. - */ const video = videoMap.get(getVideoId(item)) - if (!video) continue + if (!video) { + /** + * The item is absent from a trusted `videos.list`, so its duration is unknown + * and the Shorts decision cannot be made. Absence is NOT read as deletion: + * `isPlaylistItemPrivate` above is the only signal that removes an item from + * the listing, and dropping this one here would remove its externalId from the + * listing entirely, which is exactly how deletion reconciliation hard-deletes a + * stored document. A deferred stub is emitted instead, matching the + * include-Shorts branch: `getDocument` re-checks the video and returns null for + * a genuinely gone one, which the sync engine treats as last-known-good rather + * than as a delete. A gone video therefore costs one wasted `videos.list` per + * sync — the same cost the include-Shorts branch already pays — instead of + * silently destroying an indexed document. + */ + const stub = itemToStub(item) + if (stub) documents.push(stub) + continue + } const doc = videoToDocument(video, true) if (doc) documents.push(doc) } @@ -590,14 +611,14 @@ export const youtubeConnector: ConnectorConfig = { } const totalFetched = previouslyFetched + documents.length - if (syncContext) syncContext.totalDocsFetched = totalFetched - const hitMax = maxVideos > 0 && totalFetched >= maxVideos - if (hitMax && maxVideos > 0) { - const overflow = totalFetched - maxVideos - if (overflow > 0) documents = documents.slice(0, documents.length - overflow) - if (syncContext) syncContext.totalDocsFetched = maxVideos + + /** Videos trimmed off this page by the cap while still present at the source. */ + const trimmedByCap = hitMax ? totalFetched - maxVideos : 0 + if (trimmedByCap > 0) { + documents = documents.slice(0, documents.length - trimmedByCap) } + if (syncContext) syncContext.totalDocsFetched = hitMax ? maxVideos : totalFetched /** * Pagination is driven exclusively by the `playlistItems.list` cursor, never by how @@ -607,16 +628,18 @@ export const youtubeConnector: ConnectorConfig = { */ const nextPageToken = data.nextPageToken as string | undefined - // When the `maxVideos` cap stops the listing before the source is exhausted, mark the - // listing as capped so the sync engine does not delete still-present-but-unlisted - // videos from the knowledge base. `stopEarly` (publishedAfter cutoff) is NOT a cap — - // every remaining video is older than the cutoff and intentionally out of scope, so - // those should reconcile (delete) normally. - if (hitMax && Boolean(nextPageToken) && syncContext) { + /** + * When the `maxVideos` cap stops the listing before the source is exhausted, mark + * the listing as capped so the sync engine does not delete still-present-but-unlisted + * videos. Both truncation shapes count: videos trimmed off this page, and a further + * page left unread. The `publishedAfter` cutoff is NOT a cap — those videos are + * intentionally out of scope and must reconcile (delete) normally. + */ + if (hitMax && (trimmedByCap > 0 || nextPageToken) && syncContext) { syncContext.listingCapped = true } - const hasMore = !hitMax && !stopEarly && Boolean(nextPageToken) + const hasMore = !hitMax && Boolean(nextPageToken) return { documents, @@ -635,30 +658,31 @@ export const youtubeConnector: ConnectorConfig = { const url = `${YOUTUBE_API_BASE}/videos?part=snippet,contentDetails,status&id=${encodeURIComponent(externalId)}&key=${encodeURIComponent(apiKey)}` - try { - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { Accept: 'application/json' }, - }) + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { Accept: 'application/json' }, + }) - if (!response.ok) { - if (response.status === 403 || response.status === 404) return null - throw new Error(`Failed to get YouTube video: ${response.status}`) - } + /** + * Only `videoNotFound` (404) is a genuine absence. A 403 from `videos.list` is + * `quotaExceeded` or `forbidden` — none of the parts requested here are + * owner-restricted — so it is a fault that must surface as a failed document + * rather than silently dropping the video from the run. + */ + if (!response.ok) { + if (response.status === 404) return null + throw new Error(`Failed to get YouTube video: ${response.status}`) + } - const data = await response.json() - const items = (data.items ?? []) as VideoItem[] - const video = items[0] + const data = await response.json() + const items = (data.items ?? []) as VideoItem[] + const video = items[0] - // An empty items array means the video is deleted or private. Region-restricted - // videos are still returned here, with contentDetails.regionRestriction populated. - if (!video) return null + // An empty items array means the video is deleted or private. Region-restricted + // videos are still returned here, with contentDetails.regionRestriction populated. + if (!video) return null - return videoToDocument(video, excludeShorts) - } catch (error) { - logger.warn(`Failed to fetch YouTube video ${externalId}`, { error: toError(error).message }) - return null - } + return videoToDocument(video, excludeShorts) }, validateConfig: async ( diff --git a/apps/sim/connectors/zendesk/meta.ts b/apps/sim/connectors/zendesk/meta.ts index 87554114bc2..9e74d0f2e9b 100644 --- a/apps/sim/connectors/zendesk/meta.ts +++ b/apps/sim/connectors/zendesk/meta.ts @@ -50,7 +50,8 @@ export const zendeskConnectorMeta: ConnectorMeta = { title: 'Ticket Status Filter', type: 'dropdown', required: false, - description: 'Filter tickets by status (applies only when syncing tickets)', + description: + 'Filter tickets by status (applies only when syncing tickets). Filtering uses the Zendesk Search API, which returns at most 1,000 tickets.', options: [ { label: 'All Statuses', id: 'all' }, { label: 'New', id: 'new' }, diff --git a/apps/sim/connectors/zendesk/zendesk.test.ts b/apps/sim/connectors/zendesk/zendesk.test.ts index 0305d97f2ac..d4cb60dd8d1 100644 --- a/apps/sim/connectors/zendesk/zendesk.test.ts +++ b/apps/sim/connectors/zendesk/zendesk.test.ts @@ -1,8 +1,15 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { buildBaseUrl } from '@/connectors/zendesk/zendesk' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSecureFetch } = vi.hoisted(() => ({ mockSecureFetch: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + secureFetchWithRetry: mockSecureFetch, +})) + +import { buildBaseUrl, sameOriginNextUrl, zendeskConnector } from '@/connectors/zendesk/zendesk' describe('buildBaseUrl', () => { it.concurrent('builds the base URL for a valid subdomain', () => { @@ -58,3 +65,162 @@ describe('buildBaseUrl', () => { }) }) }) + +describe('sameOriginNextUrl', () => { + const baseUrl = 'https://acme.zendesk.com' + + it.concurrent('accepts a continuation URL on the validated base URL', () => { + const next = `${baseUrl}/api/v2/tickets.json?page%5Bafter%5D=abc` + expect(sameOriginNextUrl(next, baseUrl)).toBe(next) + }) + + it.concurrent.each([ + ['a foreign host', 'https://evil.com/api/v2/tickets.json'], + ['a host-prefix lookalike', 'https://acme.zendesk.com.evil.com/api/v2/tickets.json'], + ['a bare base URL with no path separator', 'https://acme.zendesk.com'], + ['a non-string value', 42], + ['null', null], + ['undefined', undefined], + ['an empty string', ''], + ])('rejects %s', (_label, next) => { + expect(sameOriginNextUrl(next, baseUrl)).toBeNull() + }) +}) + +const BASE = 'https://acme.zendesk.com' +const CONFIG = { subdomain: 'acme', email: 'agent@acme.com', contentType: 'tickets' } + +function ticket(id: number) { + return { + id, + subject: `Ticket ${id}`, + description: 'body', + status: 'open', + priority: 'normal', + tags: ['a'], + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-02-01T00:00:00Z', + } +} + +/** Answers each request URL with a JSON body, recording the URLs requested. */ +function mockApi(handler: (url: string) => unknown): string[] { + const urls: string[] = [] + mockSecureFetch.mockImplementation(async (url: string) => { + urls.push(url) + return { ok: true, status: 200, json: async () => handler(url) } + }) + return urls +} + +describe('zendeskConnector.listDocuments ticket capping', () => { + beforeEach(() => { + mockSecureFetch.mockReset() + }) + + it('caps the listing when the page carries more tickets than maxTickets', async () => { + mockApi(() => ({ tickets: [ticket(1), ticket(2), ticket(3)], meta: { has_more: false } })) + + const syncContext: Record = {} + const result = await zendeskConnector.listDocuments( + 'tok', + { ...CONFIG, maxTickets: '2' }, + undefined, + syncContext + ) + + // Ticket 3 still exists at the source but was trimmed, so reconciling deletions + // against this listing would hard-delete it. + expect(result.documents.map((d) => d.externalId)).toEqual(['ticket-1', 'ticket-2']) + expect(syncContext.listingCapped).toBe(true) + }) + + it('does not cap when the source is exhausted exactly at the limit', async () => { + mockApi(() => ({ tickets: [ticket(1), ticket(2)], meta: { has_more: false } })) + + const syncContext: Record = {} + const result = await zendeskConnector.listDocuments( + 'tok', + { ...CONFIG, maxTickets: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('caps when the cursor still has a further page at the limit', async () => { + mockApi(() => ({ + tickets: [ticket(1), ticket(2)], + meta: { has_more: true }, + links: { next: `${BASE}/api/v2/tickets.json?page%5Bafter%5D=x` }, + })) + + const syncContext: Record = {} + await zendeskConnector.listDocuments( + 'tok', + { ...CONFIG, maxTickets: '2' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('drains the cursor and never sends offset page params', async () => { + let page = 0 + const urls = mockApi(() => { + page += 1 + return page === 1 + ? { + tickets: [ticket(1)], + meta: { has_more: true }, + links: { next: `${BASE}/api/v2/tickets.json?page%5Bafter%5D=x` }, + } + : { tickets: [ticket(2)], meta: { has_more: false } } + }) + + const syncContext: Record = {} + const result = await zendeskConnector.listDocuments( + 'tok', + { ...CONFIG, maxTickets: '500' }, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['ticket-1', 'ticket-2']) + expect(urls[0]).toContain('page%5Bsize%5D=100') + expect(urls[0]).toContain('sort=-updated_at') + expect(syncContext.listingCapped).toBeUndefined() + }) +}) + +describe('zendeskConnector ticket contentHash invariant', () => { + beforeEach(() => { + mockSecureFetch.mockReset() + }) + + it('produces an identical contentHash from the listing stub and getDocument', async () => { + mockApi(() => ({ tickets: [ticket(7)], meta: { has_more: false } })) + const listed = await zendeskConnector.listDocuments('tok', CONFIG) + const stub = listed.documents[0] + + mockApi((url) => (url.includes('/comments') ? { comments: [] } : { ticket: ticket(7) })) + const hydrated = await zendeskConnector.getDocument('tok', CONFIG, 'ticket-7') + + expect(stub.contentDeferred).toBe(true) + expect(hydrated?.contentDeferred).toBe(false) + expect(hydrated?.contentHash).toBe(stub.contentHash) + expect(hydrated?.externalId).toBe(stub.externalId) + expect(hydrated?.sourceUrl).toBe(stub.sourceUrl) + }) + + it('returns null for a deleted ticket but rethrows a server fault', async () => { + mockSecureFetch.mockResolvedValue({ ok: false, status: 404, json: async () => ({}) }) + expect(await zendeskConnector.getDocument('tok', CONFIG, 'ticket-7')).toBeNull() + + mockSecureFetch.mockResolvedValue({ ok: false, status: 500, json: async () => ({}) }) + await expect(zendeskConnector.getDocument('tok', CONFIG, 'ticket-7')).rejects.toThrow('500') + }) +}) diff --git a/apps/sim/connectors/zendesk/zendesk.ts b/apps/sim/connectors/zendesk/zendesk.ts index d23382ee5dc..8ea3d6b89c1 100644 --- a/apps/sim/connectors/zendesk/zendesk.ts +++ b/apps/sim/connectors/zendesk/zendesk.ts @@ -8,10 +8,23 @@ import { DEFAULT_MAX_TICKETS, zendeskConnectorMeta } from '@/connectors/zendesk/ const logger = createLogger('ZendeskConnector') -const ARTICLES_PER_PAGE = 30 -const TICKETS_PER_PAGE = 100 +/** Zendesk caps cursor and offset page sizes at 100 records on these endpoints. */ +const PAGE_SIZE = 100 + +/** + * The Search API returns at most 1,000 results per query and answers 422 for any + * page beyond that depth, so ticket listings that go through it are hard-capped. + * @see https://developer.zendesk.com/api-reference/ticketing/ticket-management/search/ + */ const SEARCH_API_RESULT_CAP = 1000 +/** + * Safety valves on the unbounded cursor loops. Articles have no user-facing cap, + * so a runaway or looping cursor must not pull an unbounded corpus into memory. + */ +const MAX_ARTICLE_PAGES = 500 +const MAX_COMMENT_PAGES = 100 + const VALID_TICKET_STATUSES = new Set(['new', 'open', 'pending', 'hold', 'solved', 'closed']) interface ZendeskArticle { @@ -51,6 +64,13 @@ interface ZendeskComment { public: boolean } +/** Result of a paged fetch, carrying whether the source still had records left. */ +interface PagedResult { + items: T[] + /** True when records matching the request still exist beyond what was returned. */ + capped: boolean +} + /** * Strict Zendesk subdomain label: a single DNS label of lowercase letters, * digits, and hyphens (not leading/trailing). Rejects anything that could @@ -80,8 +100,26 @@ export function buildBaseUrl(subdomain: string): string { } /** - * Makes an authenticated GET request to the Zendesk API. - * Uses email/token authentication. + * Accepts a paginated `links.next` / `next_page` URL only when it stays on the + * validated Zendesk base URL, so a mutated or spoofed response body cannot walk + * the sync onto an arbitrary host. + */ +export function sameOriginNextUrl(nextUrl: unknown, baseUrl: string): string | null { + if (typeof nextUrl !== 'string' || nextUrl.length === 0) return null + return nextUrl.startsWith(`${baseUrl}/`) ? nextUrl : null +} + +/** Carries the HTTP status so callers can tell a deleted record from a fault. */ +class ZendeskApiError extends Error { + constructor(readonly status: number) { + super(`Zendesk API HTTP error: ${status}`) + this.name = 'ZendeskApiError' + } +} + +/** + * Makes an authenticated GET request to the Zendesk API using API-token Basic + * auth (`{email}/token:{api_token}`). */ async function zendeskApiGet( url: string, @@ -89,14 +127,17 @@ async function zendeskApiGet( sourceConfig: Record, retryOptions?: Parameters[2] ): Promise> { - const email = sourceConfig.email as string + const email = ((sourceConfig.email as string) ?? '').trim() + if (!email) { + throw new Error('Email is required for Zendesk API authentication') + } const response = await secureFetchWithRetry( url, { method: 'GET', headers: { - Authorization: `Basic ${btoa(`${email}/token:${accessToken}`)}`, + Authorization: `Basic ${Buffer.from(`${email}/token:${accessToken}`, 'utf8').toString('base64')}`, Accept: 'application/json', }, }, @@ -104,110 +145,193 @@ async function zendeskApiGet( ) if (!response.ok) { - throw new Error(`Zendesk API HTTP error: ${response.status}`) + throw new ZendeskApiError(response.status) } return (await response.json()) as Record } /** - * Fetches all Help Center articles with pagination. + * Reads the cursor-pagination continuation from a Zendesk response. Offset + * pagination is limited to the first 100 pages / 10,000 records and answers 400 + * past that depth, so every unbounded listing uses cursor pagination instead. + * @see https://developer.zendesk.com/api-reference/introduction/pagination/ + */ +function readCursorNext(data: Record, baseUrl: string): string | null { + const meta = data.meta as { has_more?: boolean } | undefined + if (meta?.has_more !== true) return null + const links = data.links as { next?: string } | undefined + return sameOriginNextUrl(links?.next, baseUrl) +} + +/** + * Fetches Help Center articles via cursor pagination. + * + * `capped` is true only if the page safety valve trips while more articles + * remain — a fully drained cursor is a complete listing. */ async function fetchArticles( - subdomain: string, + baseUrl: string, accessToken: string, sourceConfig: Record, locale?: string -): Promise { - const allArticles: ZendeskArticle[] = [] - const baseUrl = buildBaseUrl(subdomain) +): Promise> { + const items: ZendeskArticle[] = [] const localePath = locale ? `/${encodeURIComponent(locale)}` : '' - let page = 1 + const params = new URLSearchParams({ 'page[size]': String(PAGE_SIZE) }) + let url: string | null = `${baseUrl}/api/v2/help_center${localePath}/articles.json?${params}` + let pages = 0 - while (true) { - const url = `${baseUrl}/api/v2/help_center${localePath}/articles.json?page=${page}&per_page=${ARTICLES_PER_PAGE}` + while (url) { const data = await zendeskApiGet(url, accessToken, sourceConfig) - const articles = (data.articles as ZendeskArticle[]) || [] - - if (articles.length === 0) break + items.push(...((data.articles as ZendeskArticle[]) || [])) + pages += 1 - allArticles.push(...articles) - - if (!data.next_page) break - page++ + url = readCursorNext(data, baseUrl) + if (url && pages >= MAX_ARTICLE_PAGES) { + logger.warn( + `Zendesk article listing stopped at the ${MAX_ARTICLE_PAGES}-page safety valve with more articles remaining; listing is incomplete.` + ) + return { items, capped: true } + } } - return allArticles + return { items, capped: false } } /** - * Fetches tickets with optional status filtering and pagination. + * Fetches tickets, honouring the configured cap. + * + * Without a status filter this uses cursor pagination on the tickets endpoint + * (no depth cap). With a status filter it must use the Search API, which is + * offset-only and hard-capped at 1,000 results. + * + * `capped` is true only when tickets matching the request still exist beyond + * what is returned, so a genuinely exhausted source still reconciles deletions. */ async function fetchTickets( - subdomain: string, + baseUrl: string, accessToken: string, sourceConfig: Record, - statusFilter?: string, - maxTickets?: number -): Promise { - const allTickets: ZendeskTicket[] = [] - const baseUrl = buildBaseUrl(subdomain) - const limit = maxTickets || DEFAULT_MAX_TICKETS - let url: string | null = `${baseUrl}/api/v2/tickets.json?per_page=${TICKETS_PER_PAGE}` - - if (statusFilter && statusFilter !== 'all') { - if (VALID_TICKET_STATUSES.has(statusFilter)) { - if (limit > SEARCH_API_RESULT_CAP) { - logger.warn( - `Zendesk Search API caps at ${SEARCH_API_RESULT_CAP} results; requested limit ${limit} will be truncated. Remove status filter to use the unbounded tickets endpoint.` - ) - } - const params = new URLSearchParams({ - query: `type:ticket status:${statusFilter}`, - per_page: String(TICKETS_PER_PAGE), - }) - url = `${baseUrl}/api/v2/search.json?${params.toString()}` - } else { - logger.warn( - `Invalid Zendesk statusFilter "${statusFilter}"; falling back to all tickets. Valid values: ${[...VALID_TICKET_STATUSES].join(', ')}.` - ) - } + statusFilter: string | undefined, + limit: number +): Promise> { + const useSearch = VALID_TICKET_STATUSES.has(statusFilter ?? '') + + if (statusFilter && statusFilter !== 'all' && !useSearch) { + logger.warn( + `Invalid Zendesk statusFilter "${statusFilter}"; falling back to all tickets. Valid values: ${[...VALID_TICKET_STATUSES].join(', ')}.` + ) } - while (url && allTickets.length < limit) { + return useSearch + ? fetchTicketsViaSearch(baseUrl, accessToken, sourceConfig, statusFilter as string, limit) + : fetchTicketsViaCursor(baseUrl, accessToken, sourceConfig, limit) +} + +/** + * Lists tickets newest-updated first via cursor pagination, which has no page-depth + * limit. `sort=-updated_at` is one of the three sort keys the tickets endpoint accepts + * under cursor pagination (`updated_at`, `id`, `status`, each optionally `-` prefixed), + * and it is what makes the `limit` truncation keep the most recently touched tickets + * rather than an arbitrary slice by id. + */ +async function fetchTicketsViaCursor( + baseUrl: string, + accessToken: string, + sourceConfig: Record, + limit: number +): Promise> { + const items: ZendeskTicket[] = [] + const params = new URLSearchParams({ 'page[size]': String(PAGE_SIZE), sort: '-updated_at' }) + let url: string | null = `${baseUrl}/api/v2/tickets.json?${params}` + let sourceHasMore = false + + while (url) { const data = await zendeskApiGet(url, accessToken, sourceConfig) - const tickets = ((data.tickets || data.results) as ZendeskTicket[]) || [] + items.push(...((data.tickets as ZendeskTicket[]) || [])) + + url = readCursorNext(data, baseUrl) + sourceHasMore = url !== null + if (items.length >= limit) break + } + + const capped = items.length > limit || (items.length >= limit && sourceHasMore) + return { items: items.slice(0, limit), capped } +} + +async function fetchTicketsViaSearch( + baseUrl: string, + accessToken: string, + sourceConfig: Record, + statusFilter: string, + limit: number +): Promise> { + const effectiveLimit = Math.min(limit, SEARCH_API_RESULT_CAP) + if (limit > SEARCH_API_RESULT_CAP) { + logger.warn( + `Zendesk Search API caps at ${SEARCH_API_RESULT_CAP} results; requested limit ${limit} is truncated. Remove the status filter to use the unbounded tickets endpoint.` + ) + } - if (tickets.length === 0) break + const items: ZendeskTicket[] = [] + const params = new URLSearchParams({ + query: `type:ticket status:${statusFilter}`, + sort_by: 'updated_at', + sort_order: 'desc', + per_page: String(PAGE_SIZE), + }) + let url: string | null = `${baseUrl}/api/v2/search.json?${params}` + let sourceHasMore = false + let totalMatches: number | null = null - allTickets.push(...tickets) + while (url) { + const data = await zendeskApiGet(url, accessToken, sourceConfig) + items.push(...((data.results as ZendeskTicket[]) || [])) + if (typeof data.count === 'number') totalMatches = data.count - url = (data.next_page as string) || null + url = sameOriginNextUrl(data.next_page, baseUrl) + sourceHasMore = url !== null + if (items.length >= effectiveLimit) break } - return allTickets.slice(0, limit) + const returned = Math.min(items.length, effectiveLimit) + const capped = + sourceHasMore || + items.length > effectiveLimit || + (totalMatches !== null && totalMatches > returned) + + return { items: items.slice(0, effectiveLimit), capped } } /** - * Fetches all comments for a ticket. + * Fetches comments for a ticket via cursor pagination, bounded by a page + * safety valve so a pathological ticket cannot exhaust memory. */ async function fetchTicketComments( - subdomain: string, + baseUrl: string, accessToken: string, sourceConfig: Record, ticketId: number ): Promise { const allComments: ZendeskComment[] = [] - const baseUrl = buildBaseUrl(subdomain) - let url: string | null = `${baseUrl}/api/v2/tickets/${ticketId}/comments.json?per_page=100` + const params = new URLSearchParams({ 'page[size]': String(PAGE_SIZE) }) + let url: string | null = `${baseUrl}/api/v2/tickets/${ticketId}/comments.json?${params}` + let pages = 0 while (url) { const data = await zendeskApiGet(url, accessToken, sourceConfig) - const comments = (data.comments as ZendeskComment[]) || [] - - allComments.push(...comments) - - url = (data.next_page as string) || null + allComments.push(...((data.comments as ZendeskComment[]) || [])) + pages += 1 + + url = readCursorNext(data, baseUrl) + if (url && pages >= MAX_COMMENT_PAGES) { + logger.warn('Zendesk ticket comment listing truncated at the page safety valve', { + ticketId, + comments: allComments.length, + }) + break + } } return allComments @@ -226,12 +350,12 @@ function formatTicketContent(ticket: ZendeskTicket, comments: ZendeskComment[]): } parts.push(`Created: ${ticket.created_at}`) parts.push(`Updated: ${ticket.updated_at}`) - if (ticket.tags.length > 0) { + if (ticket.tags?.length) { parts.push(`Tags: ${ticket.tags.join(', ')}`) } parts.push('') parts.push('--- Description ---') - parts.push(htmlToPlainText(ticket.description)) + parts.push(htmlToPlainText(ticket.description || '')) if (comments.length > 0) { parts.push('') @@ -239,7 +363,7 @@ function formatTicketContent(ticket: ZendeskTicket, comments: ZendeskComment[]): for (const comment of comments) { const visibility = comment.public ? 'Public' : 'Internal' parts.push(`\n[${comment.created_at}] (${visibility}) Author ${comment.author_id}:`) - parts.push(htmlToPlainText(comment.html_body || comment.body)) + parts.push(htmlToPlainText(comment.html_body || comment.body || '')) } } @@ -250,15 +374,15 @@ function formatTicketContent(ticket: ZendeskTicket, comments: ZendeskComment[]): * Converts an article to an ExternalDocument with inline content. * Articles return body inline from the list API so no deferral is needed. */ -function articleToDocument(article: ZendeskArticle, subdomain: string): ExternalDocument { +function articleToDocument(article: ZendeskArticle, baseUrl: string): ExternalDocument { const content = htmlToPlainText(article.body || '') return { externalId: `article-${article.id}`, - title: article.title, + title: article.title || 'Untitled', content, mimeType: 'text/plain', - sourceUrl: article.html_url || `https://${subdomain}.zendesk.com/hc/articles/${article.id}`, + sourceUrl: article.html_url || `${baseUrl}/hc/articles/${article.id}`, contentHash: `zendesk:article:${article.id}:${article.updated_at}`, metadata: { type: 'article', @@ -279,14 +403,14 @@ function articleToDocument(article: ZendeskArticle, subdomain: string): External * each ticket requires a separate comments API call. Full content is fetched * lazily via getDocument only for new/changed documents. */ -function ticketToStub(ticket: ZendeskTicket, subdomain: string): ExternalDocument { +function ticketToStub(ticket: ZendeskTicket, baseUrl: string): ExternalDocument { return { externalId: `ticket-${ticket.id}`, - title: `Ticket #${ticket.id}: ${ticket.subject}`, + title: `Ticket #${ticket.id}: ${ticket.subject || 'Untitled'}`, content: '', contentDeferred: true, mimeType: 'text/plain', - sourceUrl: `https://${subdomain}.zendesk.com/agent/tickets/${ticket.id}`, + sourceUrl: `${baseUrl}/agent/tickets/${ticket.id}`, contentHash: `zendesk:ticket:${ticket.id}:${ticket.updated_at}`, metadata: { type: 'ticket', @@ -301,33 +425,25 @@ function ticketToStub(ticket: ZendeskTicket, subdomain: string): ExternalDocumen } /** - * Converts a ticket (with comments) to a full ExternalDocument. - * Used by getDocument to resolve deferred ticket stubs. + * Converts a ticket (with comments) to a full ExternalDocument. Used by + * getDocument to resolve deferred ticket stubs; the identity fields and + * `contentHash` are taken from the same stub so a hydrated document never + * looks changed relative to its listing entry. */ function ticketToDocument( ticket: ZendeskTicket, comments: ZendeskComment[], - subdomain: string + baseUrl: string ): ExternalDocument { - const content = formatTicketContent(ticket, comments) + const stub = ticketToStub(ticket, baseUrl) return { - externalId: `ticket-${ticket.id}`, - title: `Ticket #${ticket.id}: ${ticket.subject}`, - content, + ...stub, + content: formatTicketContent(ticket, comments), contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: `https://${subdomain}.zendesk.com/agent/tickets/${ticket.id}`, - contentHash: `zendesk:ticket:${ticket.id}:${ticket.updated_at}`, metadata: { - type: 'ticket', - ticketId: ticket.id, - status: ticket.status, - priority: ticket.priority, - tags: ticket.tags, + ...stub.metadata, commentCount: comments.length, - createdAt: ticket.created_at, - updatedAt: ticket.updated_at, }, } } @@ -339,54 +455,71 @@ export const zendeskConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { const subdomain = (sourceConfig.subdomain as string)?.trim() if (!subdomain) { throw new Error('Subdomain is required') } + const baseUrl = buildBaseUrl(subdomain) - const email = sourceConfig.email as string - if (!email?.trim()) { + const email = (sourceConfig.email as string)?.trim() + if (!email) { throw new Error('Email is required') } const contentType = (sourceConfig.contentType as string) || 'both' const ticketStatus = sourceConfig.ticketStatus as string | undefined const locale = (sourceConfig.locale as string)?.trim() || undefined - const maxTickets = sourceConfig.maxTickets - ? Number(sourceConfig.maxTickets) - : DEFAULT_MAX_TICKETS + const parsedMax = Number(sourceConfig.maxTickets) + const maxTickets = + Number.isFinite(parsedMax) && parsedMax > 0 ? Math.floor(parsedMax) : DEFAULT_MAX_TICKETS const documents: ExternalDocument[] = [] + let listingCapped = false if (contentType === 'articles' || contentType === 'both') { logger.info('Fetching Zendesk Help Center articles', { subdomain, locale }) - const articles = await fetchArticles(subdomain, accessToken, sourceConfig, locale) - logger.info(`Fetched ${articles.length} articles from Zendesk`) + const articles = await fetchArticles(baseUrl, accessToken, sourceConfig, locale) + logger.info(`Fetched ${articles.items.length} articles from Zendesk`) + listingCapped ||= articles.capped - for (const article of articles) { + for (const article of articles.items) { if (!article.body?.trim()) continue - documents.push(articleToDocument(article, subdomain)) + documents.push(articleToDocument(article, baseUrl)) } } if (contentType === 'tickets' || contentType === 'both') { logger.info('Fetching Zendesk support tickets', { subdomain, ticketStatus, maxTickets }) const tickets = await fetchTickets( - subdomain, + baseUrl, accessToken, sourceConfig, ticketStatus, maxTickets ) - logger.info(`Fetched ${tickets.length} tickets from Zendesk`) + logger.info(`Fetched ${tickets.items.length} tickets from Zendesk`) + listingCapped ||= tickets.capped - for (const ticket of tickets) { - documents.push(ticketToStub(ticket, subdomain)) + for (const ticket of tickets.items) { + documents.push(ticketToStub(ticket, baseUrl)) } } + /** + * The sync engine hard-deletes stored documents absent from a full listing. + * Flag the listing as capped whenever the source still held matching records + * beyond what was returned, so the truncated remainder is not deleted. + */ + if (listingCapped && syncContext) { + syncContext.listingCapped = true + logger.warn('Zendesk listing was capped; deletion reconciliation will be suppressed', { + subdomain, + documents: documents.length, + }) + } + return { documents, hasMore: false, @@ -399,37 +532,49 @@ export const zendeskConnector: ConnectorConfig = { externalId: string ): Promise => { const subdomain = (sourceConfig.subdomain as string)?.trim() - if (!subdomain) return null + /** + * A misconfigured source is a failure, not an absent document. Returning + * `null` reads as documented absence, which on an `add` drops the document + * with no counter and no log. `listDocuments` throws on the same condition. + */ + if (!subdomain) throw new Error('Subdomain is required') try { + const baseUrl = buildBaseUrl(subdomain) + if (externalId.startsWith('article-')) { - const articleId = externalId.replace('article-', '') - const baseUrl = buildBaseUrl(subdomain) - const url = `${baseUrl}/api/v2/help_center/articles/${articleId}.json` + const articleId = externalId.slice('article-'.length) + const url = `${baseUrl}/api/v2/help_center/articles/${encodeURIComponent(articleId)}.json` const data = await zendeskApiGet(url, accessToken, sourceConfig) const article = data.article as ZendeskArticle if (!article) return null - return articleToDocument(article, subdomain) + return articleToDocument(article, baseUrl) } if (externalId.startsWith('ticket-')) { - const ticketId = Number(externalId.replace('ticket-', '')) - const baseUrl = buildBaseUrl(subdomain) + const ticketId = Number(externalId.slice('ticket-'.length)) + if (!Number.isInteger(ticketId) || ticketId <= 0) return null const url = `${baseUrl}/api/v2/tickets/${ticketId}.json` const data = await zendeskApiGet(url, accessToken, sourceConfig) const ticket = data.ticket as ZendeskTicket if (!ticket) return null - const comments = await fetchTicketComments(subdomain, accessToken, sourceConfig, ticketId) - return ticketToDocument(ticket, comments, subdomain) + const comments = await fetchTicketComments(baseUrl, accessToken, sourceConfig, ticketId) + return ticketToDocument(ticket, comments, baseUrl) } return null } catch (error) { - logger.warn('Failed to get Zendesk document', { - externalId, - error: toError(error).message, - }) - return null + /** + * A deleted record (404) is the only documented absence and resolves to + * `null`. Every other failure is rethrown so the sync engine records a + * visible failed document instead of dropping it from the run with no + * counter — and, for an already-indexed ticket, keeps it as + * last-known-good rather than exposing it to deletion reconciliation. + */ + if (error instanceof ZendeskApiError && error.status === 404) { + return null + } + throw toError(error) } }, diff --git a/apps/sim/connectors/zoho-desk/zoho-desk.ts b/apps/sim/connectors/zoho-desk/zoho-desk.ts index badbd6a2c39..c1b3c5d5eca 100644 --- a/apps/sim/connectors/zoho-desk/zoho-desk.ts +++ b/apps/sim/connectors/zoho-desk/zoho-desk.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' @@ -40,6 +40,9 @@ const MAX_CONVERSATION_ENTRIES = 400 */ const MAX_HYDRATED_THREADS = 50 +/** Thread bodies fetched in parallel per ticket. Kept low to stay inside Zoho's per-minute API credit budget. */ +const THREAD_FETCH_CONCURRENCY = 4 + interface ZohoDeskArticleSummary { id: string title?: string @@ -158,6 +161,17 @@ function resolveMax(value: unknown, fallback: number, label: string): number { return Math.floor(parsed) } +/** Carries the HTTP status so callers can tell a deleted record from a fault. */ +class ZohoDeskApiError extends Error { + constructor( + readonly status: number, + message: string + ) { + super(message) + this.name = 'ZohoDeskApiError' + } +} + /** * Performs an authenticated GET against the Zoho Desk API. * @@ -189,7 +203,7 @@ async function deskGet( : typeof body.errorCode === 'string' && body.errorCode.trim() ? body.errorCode : `Zoho Desk API HTTP error: ${response.status}` - throw new Error(message) + throw new ZohoDeskApiError(response.status, message) } return body @@ -200,17 +214,26 @@ function readDataArray(body: Record): T[] { return Array.isArray(body.data) ? (body.data as T[]) : [] } +/** Matches a value that carries HTML markup rather than plain prose. */ +const HTML_MARKUP_PATTERN = /<[a-z][\s\S]*>/i + /** * Renders a Zoho content value as plain text. Zoho spells the HTML discriminator - * both as `html` (comments) and `text/html` (threads), so both are matched; a - * value without an HTML content type is passed through untouched so genuinely - * plain bodies are not mangled by tag stripping. + * both as `html` (comments) and `text/html` (threads), so both are matched. + * + * A ticket's `description` and `resolution` carry no content-type at all while + * still holding rich text, so an undeclared value is sniffed for markup and + * stripped only when tags are actually present — a genuinely plain body is + * never mangled. */ function toPlainText(content: string | undefined, contentType: string | undefined): string { if (!content) return '' const normalized = contentType?.trim().toLowerCase() ?? '' - const isHtml = normalized === 'html' || normalized.startsWith('text/html') - return isHtml ? htmlToPlainText(content) : content + if (normalized) { + const isHtml = normalized === 'html' || normalized.startsWith('text/html') + return isHtml ? htmlToPlainText(content) : content + } + return HTML_MARKUP_PATTERN.test(content) ? htmlToPlainText(content) : content } /** @@ -414,13 +437,7 @@ function formatTicketContent(ticket: ZohoDeskTicket, conversation: string[]): st if (ticket.description) { parts.push('') parts.push('--- Description ---') - // Zoho ships no content-type discriminator for `description`, and it can be - // either HTML or plain text, so strip only when markup is actually present. - parts.push( - /<[a-z][\s\S]*>/i.test(ticket.description) - ? htmlToPlainText(ticket.description) - : ticket.description - ) + parts.push(toPlainText(ticket.description, undefined)) } // The agent's resolution note is the single most reusable answer on a ticket, @@ -428,11 +445,7 @@ function formatTicketContent(ticket: ZohoDeskTicket, conversation: string[]): st if (ticket.resolution) { parts.push('') parts.push('--- Resolution ---') - parts.push( - /<[a-z][\s\S]*>/i.test(ticket.resolution) - ? htmlToPlainText(ticket.resolution) - : ticket.resolution - ) + parts.push(toPlainText(ticket.resolution, undefined)) } if (conversation.length > 0) { @@ -648,26 +661,36 @@ export const zohoDeskConnector: ConnectorConfig = { cap: MAX_CONVERSATION_ENTRIES, }) } - const blocks: string[] = [] - let hydratedThreads = 0 - - for (const entry of entries) { - let hydrated: ZohoDeskThreadDetail | null = null - if (entry.type === 'thread' && hydratedThreads < MAX_HYDRATED_THREADS) { - try { - hydrated = await fetchThread(apiBase, accessToken, orgId, ticketId, entry.id) - hydratedThreads += 1 - } catch (error) { - logger.warn('Failed to fetch Zoho Desk thread body; using summary', { - ticketId, - threadId: entry.id, - error: getErrorMessage(error), - }) - } - } - blocks.push(formatConversationEntry(entry, hydrated)) + const threadIds = entries + .filter((entry) => entry.type === 'thread') + .slice(0, MAX_HYDRATED_THREADS) + .map((entry) => entry.id) + const hydratedById = new Map() + + for (let i = 0; i < threadIds.length; i += THREAD_FETCH_CONCURRENCY) { + await Promise.all( + threadIds.slice(i, i + THREAD_FETCH_CONCURRENCY).map(async (threadId) => { + try { + const detail = await fetchThread(apiBase, accessToken, orgId, ticketId, threadId) + if (detail) hydratedById.set(threadId, detail) + } catch (error) { + logger.warn('Failed to fetch Zoho Desk thread body; using summary', { + ticketId, + threadId, + error: getErrorMessage(error), + }) + } + }) + ) } + const blocks = entries.map((entry) => + formatConversationEntry( + entry, + entry.type === 'thread' ? (hydratedById.get(entry.id) ?? null) : null + ) + ) + const content = formatTicketContent(ticket, blocks) if (!content.trim()) return null @@ -676,11 +699,15 @@ export const zohoDeskConnector: ConnectorConfig = { return null } catch (error) { - logger.warn('Failed to get Zoho Desk document', { - externalId, - error: toError(error).message, - }) - return null + /** + * Only a deleted record (404/410) resolves to `null`. Every other failure is + * rethrown so the sync engine records a visible failed document instead of + * dropping it from the run with no counter and no error log. + */ + if (error instanceof ZohoDeskApiError && (error.status === 404 || error.status === 410)) { + return null + } + throw error } }, @@ -709,7 +736,7 @@ export const zohoDeskConnector: ConnectorConfig = { resolveMax(sourceConfig.maxTickets, DEFAULT_MAX_TICKETS, 'Max tickets') resolveMax(sourceConfig.maxArticles, DEFAULT_MAX_ARTICLES, 'Max articles') } catch (error) { - return { valid: false, error: toError(error).message } + return { valid: false, error: getErrorMessage(error) } } try { @@ -732,7 +759,7 @@ export const zohoDeskConnector: ConnectorConfig = { } return { valid: true } } catch (error) { - return { valid: false, error: toError(error).message || 'Failed to validate configuration' } + return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') } } }, diff --git a/apps/sim/connectors/zoom/meta.ts b/apps/sim/connectors/zoom/meta.ts index 2be71a6a385..39be56551e4 100644 --- a/apps/sim/connectors/zoom/meta.ts +++ b/apps/sim/connectors/zoom/meta.ts @@ -12,7 +12,6 @@ export const zoomConnectorMeta: ConnectorMeta = { mode: 'oauth', provider: 'zoom', requiredScopes: [ - 'user:read:user', 'cloud_recording:read:list_user_recordings', 'cloud_recording:read:list_recording_files', ], @@ -31,8 +30,7 @@ export const zoomConnectorMeta: ConnectorMeta = { { label: 'Last 90 days', id: '90' }, { label: 'Last 6 months (recommended)', id: '180' }, ], - description: - 'On initial sync only. Zoom only allows access to cloud recordings within the last 6 months.', + description: 'How far back to sync on the first run. Later syncs only fetch new recordings.', }, { id: 'maxRecordings', diff --git a/apps/sim/connectors/zoom/zoom.test.ts b/apps/sim/connectors/zoom/zoom.test.ts index 040de394fbf..decac962516 100644 --- a/apps/sim/connectors/zoom/zoom.test.ts +++ b/apps/sim/connectors/zoom/zoom.test.ts @@ -1,8 +1,16 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { parseVtt } from '@/connectors/zoom/zoom' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) + +import { parseVtt, zoomConnector } from '@/connectors/zoom/zoom' const HEADER = 'WEBVTT\n\n' @@ -85,3 +93,96 @@ describe('parseVtt', () => { expect(result).not.toMatch(/<\/?[^>]+>/) }) }) + +function recording(uuid: string) { + return { + uuid, + id: 1, + topic: `Meeting ${uuid}`, + start_time: '2024-01-01T00:00:00Z', + recording_files: [ + { + id: `f-${uuid}`, + file_type: 'TRANSCRIPT', + status: 'completed', + file_size: 1024, + download_url: `https://acme.zoom.us/rec/download/${uuid}`, + }, + ], + } +} + +/** Answers the recordings listing with a fixed body. */ +function mockListing(body: Record) { + mockFetchWithRetry.mockResolvedValue({ + ok: true, + status: 200, + json: async () => body, + }) +} + +describe('zoomConnector.listDocuments cap accounting', () => { + beforeEach(() => { + mockFetchWithRetry.mockReset() + }) + + it('does not cap the listing when the source is exhausted exactly at the limit', async () => { + mockListing({ meetings: [recording('a'), recording('b')] }) + + const syncContext: Record = {} + const result = await zoomConnector.listDocuments( + 'tok', + { maxRecordings: '2', lookback: '30' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(result.hasMore).toBe(false) + // Nothing was withheld, so deletion reconciliation must stay enabled. + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('caps the listing when recordings are trimmed off the page', async () => { + mockListing({ meetings: [recording('a'), recording('b'), recording('c')] }) + + const syncContext: Record = {} + const result = await zoomConnector.listDocuments( + 'tok', + { maxRecordings: '2', lookback: '30' }, + undefined, + syncContext + ) + + expect(result.documents.map((d) => d.externalId)).toEqual(['a', 'b']) + expect(syncContext.listingCapped).toBe(true) + }) + + it('caps the listing when a further page remains behind the cap', async () => { + mockListing({ meetings: [recording('a'), recording('b')], next_page_token: 'tok-2' }) + + const syncContext: Record = {} + await zoomConnector.listDocuments( + 'tok', + { maxRecordings: '2', lookback: '30' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) + + it('caps the listing when a further window remains behind the cap', async () => { + mockListing({ meetings: [recording('a'), recording('b')] }) + + const syncContext: Record = {} + await zoomConnector.listDocuments( + 'tok', + { maxRecordings: '2', lookback: '180' }, + undefined, + syncContext + ) + + expect(syncContext.listingCapped).toBe(true) + }) +}) diff --git a/apps/sim/connectors/zoom/zoom.ts b/apps/sim/connectors/zoom/zoom.ts index 77122bd3fa5..0bba34df117 100644 --- a/apps/sim/connectors/zoom/zoom.ts +++ b/apps/sim/connectors/zoom/zoom.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { @@ -85,13 +85,35 @@ function encodeMeetingUuid(uuid: string): string { return encoded } +/** + * Formats a date as `yyyy-mm-dd` in UTC. Zoom documents `from`/`to` on the + * recordings listing as UTC dates, so local-timezone getters would shift the + * window boundary by a day on any non-UTC host. + */ function formatDate(date: Date): string { - const y = date.getFullYear() - const m = String(date.getMonth() + 1).padStart(2, '0') - const d = String(date.getDate()).padStart(2, '0') + const y = date.getUTCFullYear() + const m = String(date.getUTCMonth() + 1).padStart(2, '0') + const d = String(date.getUTCDate()).padStart(2, '0') return `${y}-${m}-${d}` } +/** + * True when `url` is an https URL on a Zoom-owned host. `download_url` is taken + * verbatim from an API response and is fetched with the user's access token in + * the Authorization header, so the host is checked before the token is attached. + */ +function isZoomDownloadUrl(url: string): boolean { + try { + const parsed = new URL(url) + return ( + parsed.protocol === 'https:' && + (parsed.hostname === 'zoom.us' || parsed.hostname.endsWith('.zoom.us')) + ) + } catch { + return false + } +} + function encodeCursor(state: CursorState): string { return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url') } @@ -264,10 +286,16 @@ export const zoomConnector: ConnectorConfig = { return { documents: [], hasMore: false } } + /** + * Zoom caps the `from`/`to` range at one month and both bounds are inclusive, + * so each window spans exactly `WINDOW_DAYS` calendar days (`to - (WINDOW_DAYS - 1)`). + * Consecutive windows abut without overlapping, which keeps every request + * inside the documented range and stops the seam day being listed twice. + */ const now = new Date() const earliest = new Date(now.getTime() - lookbackDays * MS_PER_DAY) const toDate = new Date(now.getTime() - state.windowIndex * WINDOW_DAYS * MS_PER_DAY) - const rawFromDate = new Date(toDate.getTime() - WINDOW_DAYS * MS_PER_DAY) + const rawFromDate = new Date(toDate.getTime() - (WINDOW_DAYS - 1) * MS_PER_DAY) const fromDate = rawFromDate < earliest ? earliest : rawFromDate if (fromDate >= toDate) { @@ -339,7 +367,23 @@ export const zoomConnector: ConnectorConfig = { const totalFetched = prevFetched + indexableCount if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = capReached - if (hitLimit && syncContext) syncContext.listingCapped = true + + /** + * `capReached` alone is not truncation. It is also true when the cap lands exactly on + * the last recording of the last window, where nothing was withheld — and + * `takeIndexableWithinCap` cannot distinguish the two, because it reports only that + * the budget ran out, not whether it stopped mid-page. Setting `listingCapped` there + * would suppress deletion reconciliation on every subsequent sync of a source that + * simply sits at its cap, so purged recordings would never leave the knowledge base. + * + * The listing is capped only when the cap actually withheld something: recordings + * dropped from this page, or a page/window left unread behind it. + */ + const droppedFromPage = documents.length < allDocuments.length + const moreBehindCap = Boolean(nextPageToken) || state.windowIndex + 1 < numWindows + if (hitLimit && (droppedFromPage || moreBehindCap) && syncContext) { + syncContext.listingCapped = true + } let nextCursor: string | undefined let hasMore = false @@ -362,81 +406,94 @@ export const zoomConnector: ConnectorConfig = { _sourceConfig: Record, externalId: string ): Promise => { - try { - if (!externalId) return null + if (!externalId) return null - const url = `${ZOOM_API_BASE}/meetings/${encodeMeetingUuid(externalId)}/recordings` + const url = `${ZOOM_API_BASE}/meetings/${encodeMeetingUuid(externalId)}/recordings` - const response = await fetchWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + const response = await fetchWithRetry(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) - if (!response.ok) { - if (response.status === 404 || response.status === 410) return null - throw new Error(`Failed to fetch Zoom recording: ${response.status}`) - } + /** + * Only a deleted recording (404/410) resolves to `null`. Every other failure + * throws so the sync engine records a visible failed document instead of + * dropping the recording from the run with no counter and no error log. + */ + if (!response.ok) { + if (response.status === 404 || response.status === 410) return null + throw new Error(`Failed to fetch Zoom recording: ${response.status}`) + } - const recording = (await response.json()) as ZoomRecording - const transcript = findTranscriptFile(recording.recording_files) + const recording = (await response.json()) as ZoomRecording + const transcript = findTranscriptFile(recording.recording_files) - if (!transcript?.download_url) { - logger.info('Transcript no longer available for Zoom recording', { externalId }) - return null - } + if (!transcript?.download_url) { + logger.info('Transcript no longer available for Zoom recording', { externalId }) + return null + } - const vttResponse = await fetchWithRetry(transcript.download_url, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}` }, - }) + if (!isZoomDownloadUrl(transcript.download_url)) { + throw new Error('Refusing to send the Zoom access token to a non-Zoom download host') + } - if (!vttResponse.ok) { - logger.warn('Failed to download Zoom transcript', { - externalId, - status: vttResponse.status, - }) + /** + * The access token goes in the Authorization header, never in the URL: Zoom + * removed support for token values in query parameters in February 2023 and + * answers `?access_token=` with `Invalid access token, this access token is + * not supported as a query parameter string`. + */ + const vttResponse = await fetchWithRetry(transcript.download_url, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, + }) + + /** + * A transcript the API just advertised is expected to download. A 404/410 means + * it was purged between the two calls; anything else is a fault and must fail + * the document rather than silently drop it. + */ + if (!vttResponse.ok) { + if (vttResponse.status === 404 || vttResponse.status === 410) { + logger.info('Zoom transcript was purged before it could be downloaded', { externalId }) return null } + throw new Error(`Failed to download Zoom transcript: ${vttResponse.status}`) + } - const vttBuffer = await readBodyWithLimit(vttResponse, CONNECTOR_MAX_FILE_BYTES) - if (!vttBuffer) { - return markSkipped( - recordingToStub(recording, transcript), - sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) - ) - } + const vttBuffer = await readBodyWithLimit(vttResponse, CONNECTOR_MAX_FILE_BYTES) + if (!vttBuffer) { + return markSkipped( + recordingToStub(recording, transcript), + sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) + ) + } - const vttText = vttBuffer.toString('utf8') - const transcriptText = parseVtt(vttText).trim() - if (!transcriptText) return null - - const content = formatTranscriptContent(recording, transcriptText) - - return { - externalId: recording.uuid || externalId, - title: recording.topic?.trim() || 'Untitled Zoom Meeting', - content, - contentDeferred: false, - mimeType: 'text/plain', - sourceUrl: buildSourceUrl(recording), - contentHash: buildContentHash(recording, transcript), - metadata: { - meetingId: recording.id != null ? String(recording.id) : undefined, - hostEmail: recording.host_email, - duration: recording.duration, - meetingDate: recording.start_time, - topic: recording.topic, - }, - } - } catch (error) { - logger.warn('Failed to get Zoom recording', { - externalId, - error: toError(error).message, - }) - return null + const vttText = vttBuffer.toString('utf8') + const transcriptText = parseVtt(vttText).trim() + if (!transcriptText) return null + + const content = formatTranscriptContent(recording, transcriptText) + + return { + externalId: recording.uuid || externalId, + title: recording.topic?.trim() || 'Untitled Zoom Meeting', + content, + contentDeferred: false, + mimeType: 'text/plain', + sourceUrl: buildSourceUrl(recording), + contentHash: buildContentHash(recording, transcript), + metadata: { + meetingId: recording.id != null ? String(recording.id) : undefined, + hostEmail: recording.host_email, + duration: recording.duration, + meetingDate: recording.start_time, + topic: recording.topic, + fileSize: transcript.file_size, + }, } }, @@ -449,9 +506,17 @@ export const zoomConnector: ConnectorConfig = { return { valid: false, error: 'Max recordings must be a non-negative number' } } + /** + * Lists a single day with `page_size=1` — the cheapest call that exercises + * the cloud-recording listing scope the sync actually depends on. `/users/me` + * would only prove the token is live, not that recordings are readable. + */ + const today = formatDate(new Date()) + const probe = new URLSearchParams({ page_size: '1', from: today, to: today }) + try { const response = await fetchWithRetry( - `${ZOOM_API_BASE}/users/me`, + `${ZOOM_API_BASE}/users/me/recordings?${probe.toString()}`, { method: 'GET', headers: { diff --git a/apps/sim/lib/api/contracts/tools/evernote.ts b/apps/sim/lib/api/contracts/tools/evernote.ts deleted file mode 100644 index 5039f00928f..00000000000 --- a/apps/sim/lib/api/contracts/tools/evernote.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const evernoteSuccessOutputSchema = (output: T) => - z.object({ - success: z.literal(true), - output, - }) - -const evernoteNoteResponseSchema = evernoteSuccessOutputSchema(z.object({ note: z.unknown() })) -const evernoteNotebookResponseSchema = evernoteSuccessOutputSchema( - z.object({ notebook: z.unknown() }) -) -const evernoteTagResponseSchema = evernoteSuccessOutputSchema(z.object({ tag: z.unknown() })) -const evernoteListNotebooksResponseSchema = evernoteSuccessOutputSchema( - z.object({ notebooks: z.array(z.unknown()) }) -) -const evernoteListTagsResponseSchema = evernoteSuccessOutputSchema( - z.object({ tags: z.array(z.unknown()) }) -) -const evernoteSearchNotesResponseSchema = evernoteSuccessOutputSchema( - z.object({ - totalNotes: z.number(), - notes: z.array(z.unknown()), - }) -) -const evernoteDeleteNoteResponseSchema = evernoteSuccessOutputSchema( - z.object({ - success: z.literal(true), - noteGuid: z.string(), - }) -) - -const CREATE_NOTE_REQUIRED = 'apiKey, title, and content are required' -export const evernoteCreateNoteBodySchema = z.object({ - apiKey: z.string({ error: CREATE_NOTE_REQUIRED }).min(1, CREATE_NOTE_REQUIRED), - title: z.string({ error: CREATE_NOTE_REQUIRED }).min(1, CREATE_NOTE_REQUIRED), - content: z.string({ error: CREATE_NOTE_REQUIRED }).min(1, CREATE_NOTE_REQUIRED), - notebookGuid: z.string().nullish(), - tagNames: z.union([z.string(), z.array(z.string())]).nullish(), -}) - -const UPDATE_NOTE_REQUIRED = 'apiKey and noteGuid are required' -export const evernoteUpdateNoteBodySchema = z.object({ - apiKey: z.string({ error: UPDATE_NOTE_REQUIRED }).min(1, UPDATE_NOTE_REQUIRED), - noteGuid: z.string({ error: UPDATE_NOTE_REQUIRED }).min(1, UPDATE_NOTE_REQUIRED), - title: z.string().nullish(), - content: z.string().nullish(), - notebookGuid: z.string().nullish(), - tagNames: z.union([z.string(), z.array(z.string())]).nullish(), -}) - -const CREATE_TAG_REQUIRED = 'apiKey and name are required' -export const evernoteCreateTagBodySchema = z.object({ - apiKey: z.string({ error: CREATE_TAG_REQUIRED }).min(1, CREATE_TAG_REQUIRED), - name: z.string({ error: CREATE_TAG_REQUIRED }).min(1, CREATE_TAG_REQUIRED), - parentGuid: z.string().nullish(), -}) - -const SEARCH_NOTES_REQUIRED = 'apiKey and query are required' -export const evernoteSearchNotesBodySchema = z.object({ - apiKey: z.string({ error: SEARCH_NOTES_REQUIRED }).min(1, SEARCH_NOTES_REQUIRED), - query: z.string({ error: SEARCH_NOTES_REQUIRED }).min(1, SEARCH_NOTES_REQUIRED), - notebookGuid: z.string().nullish(), - offset: z.unknown().optional().default(0), - maxNotes: z.unknown().optional().default(25), -}) - -const CREATE_NOTEBOOK_REQUIRED = 'apiKey and name are required' -export const evernoteCreateNotebookBodySchema = z.object({ - apiKey: z.string({ error: CREATE_NOTEBOOK_REQUIRED }).min(1, CREATE_NOTEBOOK_REQUIRED), - name: z.string({ error: CREATE_NOTEBOOK_REQUIRED }).min(1, CREATE_NOTEBOOK_REQUIRED), - stack: z.string().nullish(), -}) - -const DELETE_NOTE_REQUIRED = 'apiKey and noteGuid are required' -export const evernoteDeleteNoteBodySchema = z.object({ - apiKey: z.string({ error: DELETE_NOTE_REQUIRED }).min(1, DELETE_NOTE_REQUIRED), - noteGuid: z.string({ error: DELETE_NOTE_REQUIRED }).min(1, DELETE_NOTE_REQUIRED), -}) - -const LIST_NOTEBOOKS_REQUIRED = 'apiKey is required' -export const evernoteListNotebooksBodySchema = z.object({ - apiKey: z.string({ error: LIST_NOTEBOOKS_REQUIRED }).min(1, LIST_NOTEBOOKS_REQUIRED), -}) - -const GET_NOTEBOOK_REQUIRED = 'apiKey and notebookGuid are required' -export const evernoteGetNotebookBodySchema = z.object({ - apiKey: z.string({ error: GET_NOTEBOOK_REQUIRED }).min(1, GET_NOTEBOOK_REQUIRED), - notebookGuid: z.string({ error: GET_NOTEBOOK_REQUIRED }).min(1, GET_NOTEBOOK_REQUIRED), -}) - -const LIST_TAGS_REQUIRED = 'apiKey is required' -export const evernoteListTagsBodySchema = z.object({ - apiKey: z.string({ error: LIST_TAGS_REQUIRED }).min(1, LIST_TAGS_REQUIRED), -}) - -const GET_NOTE_REQUIRED = 'apiKey and noteGuid are required' -export const evernoteGetNoteBodySchema = z.object({ - apiKey: z.string({ error: GET_NOTE_REQUIRED }).min(1, GET_NOTE_REQUIRED), - noteGuid: z.string({ error: GET_NOTE_REQUIRED }).min(1, GET_NOTE_REQUIRED), - withContent: z.boolean().nullish(), -}) - -const COPY_NOTE_REQUIRED = 'apiKey, noteGuid, and toNotebookGuid are required' -export const evernoteCopyNoteBodySchema = z.object({ - apiKey: z.string({ error: COPY_NOTE_REQUIRED }).min(1, COPY_NOTE_REQUIRED), - noteGuid: z.string({ error: COPY_NOTE_REQUIRED }).min(1, COPY_NOTE_REQUIRED), - toNotebookGuid: z.string({ error: COPY_NOTE_REQUIRED }).min(1, COPY_NOTE_REQUIRED), -}) - -export const evernoteCreateNoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/create-note', - body: evernoteCreateNoteBodySchema, - response: { mode: 'json', schema: evernoteNoteResponseSchema }, -}) - -export const evernoteUpdateNoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/update-note', - body: evernoteUpdateNoteBodySchema, - response: { mode: 'json', schema: evernoteNoteResponseSchema }, -}) - -export const evernoteCreateTagContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/create-tag', - body: evernoteCreateTagBodySchema, - response: { mode: 'json', schema: evernoteTagResponseSchema }, -}) - -export const evernoteSearchNotesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/search-notes', - body: evernoteSearchNotesBodySchema, - response: { mode: 'json', schema: evernoteSearchNotesResponseSchema }, -}) - -export const evernoteCreateNotebookContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/create-notebook', - body: evernoteCreateNotebookBodySchema, - response: { mode: 'json', schema: evernoteNotebookResponseSchema }, -}) - -export const evernoteDeleteNoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/delete-note', - body: evernoteDeleteNoteBodySchema, - response: { mode: 'json', schema: evernoteDeleteNoteResponseSchema }, -}) - -export const evernoteListNotebooksContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/list-notebooks', - body: evernoteListNotebooksBodySchema, - response: { mode: 'json', schema: evernoteListNotebooksResponseSchema }, -}) - -export const evernoteGetNotebookContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/get-notebook', - body: evernoteGetNotebookBodySchema, - response: { mode: 'json', schema: evernoteNotebookResponseSchema }, -}) - -export const evernoteListTagsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/list-tags', - body: evernoteListTagsBodySchema, - response: { mode: 'json', schema: evernoteListTagsResponseSchema }, -}) - -export const evernoteGetNoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/get-note', - body: evernoteGetNoteBodySchema, - response: { mode: 'json', schema: evernoteNoteResponseSchema }, -}) - -export const evernoteCopyNoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/evernote/copy-note', - body: evernoteCopyNoteBodySchema, - response: { mode: 'json', schema: evernoteNoteResponseSchema }, -}) - -export type EvernoteCreateNoteBody = ContractBody -export type EvernoteCreateNoteBodyInput = ContractBodyInput -export type EvernoteCreateNoteResponse = ContractJsonResponse -export type EvernoteUpdateNoteBody = ContractBody -export type EvernoteUpdateNoteBodyInput = ContractBodyInput -export type EvernoteUpdateNoteResponse = ContractJsonResponse -export type EvernoteCreateTagBody = ContractBody -export type EvernoteCreateTagBodyInput = ContractBodyInput -export type EvernoteCreateTagResponse = ContractJsonResponse -export type EvernoteSearchNotesBody = ContractBody -export type EvernoteSearchNotesBodyInput = ContractBodyInput -export type EvernoteSearchNotesResponse = ContractJsonResponse -export type EvernoteCreateNotebookBody = ContractBody -export type EvernoteCreateNotebookBodyInput = ContractBodyInput< - typeof evernoteCreateNotebookContract -> -export type EvernoteCreateNotebookResponse = ContractJsonResponse< - typeof evernoteCreateNotebookContract -> -export type EvernoteDeleteNoteBody = ContractBody -export type EvernoteDeleteNoteBodyInput = ContractBodyInput -export type EvernoteDeleteNoteResponse = ContractJsonResponse -export type EvernoteListNotebooksBody = ContractBody -export type EvernoteListNotebooksBodyInput = ContractBodyInput -export type EvernoteListNotebooksResponse = ContractJsonResponse< - typeof evernoteListNotebooksContract -> -export type EvernoteGetNotebookBody = ContractBody -export type EvernoteGetNotebookBodyInput = ContractBodyInput -export type EvernoteGetNotebookResponse = ContractJsonResponse -export type EvernoteListTagsBody = ContractBody -export type EvernoteListTagsBodyInput = ContractBodyInput -export type EvernoteListTagsResponse = ContractJsonResponse -export type EvernoteGetNoteBody = ContractBody -export type EvernoteGetNoteBodyInput = ContractBodyInput -export type EvernoteGetNoteResponse = ContractJsonResponse -export type EvernoteCopyNoteBody = ContractBody -export type EvernoteCopyNoteBodyInput = ContractBodyInput -export type EvernoteCopyNoteResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/index.ts b/apps/sim/lib/api/contracts/tools/index.ts index fcf6dd55d7b..94ed2bb939b 100644 --- a/apps/sim/lib/api/contracts/tools/index.ts +++ b/apps/sim/lib/api/contracts/tools/index.ts @@ -11,7 +11,6 @@ export * from './databases' export * from './daytona' export * from './deployments' export * from './docusign' -export * from './evernote' export * from './file' export * from './firecrawl' export * from './fireflies' diff --git a/apps/sim/lib/auth/connectors/providers.ts b/apps/sim/lib/auth/connectors/providers.ts index 041b0c35ec8..7db19580df6 100644 --- a/apps/sim/lib/auth/connectors/providers.ts +++ b/apps/sim/lib/auth/connectors/providers.ts @@ -13,9 +13,12 @@ import { readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' import { getBaseUrl } from '@/lib/core/utils/urls' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { getMicrosoftUserInfoFromIdToken } from '@/lib/oauth/microsoft' import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' +import { MONDAY_API_URL, MONDAY_API_VERSION } from '@/tools/monday/utils' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' import { deriveZohoDeskBaseFromApiDomain } from '@/tools/zoho_desk/host-allowlist' /** @@ -1530,11 +1533,11 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/monday`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.monday.com/v2', { + const response = await fetch(MONDAY_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'API-Version': '2024-10', + 'API-Version': MONDAY_API_VERSION, Authorization: tokens.accessToken ?? '', }, body: JSON.stringify({ query: '{ me { id name email } }' }), @@ -1588,7 +1591,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { const response = await fetch('https://oauth.reddit.com/api/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, - 'User-Agent': 'sim-studio/1.0', + 'User-Agent': REDDIT_USER_AGENT, }, }) @@ -2319,9 +2322,9 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { providerId: 'docusign', clientId: env.DOCUSIGN_CLIENT_ID as string, clientSecret: env.DOCUSIGN_CLIENT_SECRET as string, - authorizationUrl: 'https://account-d.docusign.com/oauth/auth', - tokenUrl: 'https://account-d.docusign.com/oauth/token', - userInfoUrl: 'https://account-d.docusign.com/oauth/userinfo', + authorizationUrl: getDocusignOAuthUrl('/oauth/auth'), + tokenUrl: getDocusignOAuthUrl('/oauth/token'), + userInfoUrl: getDocusignOAuthUrl('/oauth/userinfo'), scopes: getCanonicalScopesForProvider('docusign'), responseType: 'code', accessType: 'offline', @@ -2331,7 +2334,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching DocuSign user profile') - const response = await fetch('https://account-d.docusign.com/oauth/userinfo', { + const response = await fetch(getDocusignOAuthUrl('/oauth/userinfo'), { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 4ef32b4279f..5a8fed1bb40 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -438,6 +438,7 @@ export const env = createEnv({ DISCORD_CLIENT_SECRET: z.string().optional(), // Discord OAuth client secret DOCUSIGN_CLIENT_ID: z.string().optional(), // DocuSign OAuth client ID DOCUSIGN_CLIENT_SECRET: z.string().optional(), // DocuSign OAuth client secret + DOCUSIGN_AUTH_HOST: z.string().optional(), // DocuSign auth host: account-d.docusign.com (demo, default) or account.docusign.com (production) MICROSOFT_CLIENT_ID: z.string().optional(), // Microsoft OAuth client ID for Office 365/Teams MICROSOFT_CLIENT_SECRET: z.string().optional(), // Microsoft OAuth client secret HUBSPOT_CLIENT_ID: z.string().optional(), // HubSpot OAuth client ID diff --git a/apps/sim/lib/integrations/icon-mapping.ts b/apps/sim/lib/integrations/icon-mapping.ts index 716dafa2781..d8849840318 100644 --- a/apps/sim/lib/integrations/icon-mapping.ts +++ b/apps/sim/lib/integrations/icon-mapping.ts @@ -72,7 +72,6 @@ import { EnrichmentIcon, EnrichSoIcon, EnrowIcon, - EvernoteIcon, ExaAIIcon, ExtendIcon, FathomIcon, @@ -329,7 +328,6 @@ export const blockTypeToIconMap: Record = { enrich: EnrichSoIcon, enrichment: EnrichmentIcon, enrow: EnrowIcon, - evernote: EvernoteIcon, exa: ExaAIIcon, extend_v2: ExtendIcon, fathom: FathomIcon, diff --git a/apps/sim/lib/integrations/integrations.json b/apps/sim/lib/integrations/integrations.json index df98e42145b..74423e143c6 100644 --- a/apps/sim/lib/integrations/integrations.json +++ b/apps/sim/lib/integrations/integrations.json @@ -6704,69 +6704,6 @@ "integrationType": "sales", "tags": ["enrichment", "sales-engagement"] }, - { - "type": "evernote", - "slug": "evernote", - "name": "Evernote", - "description": "Manage notes, notebooks, and tags in Evernote", - "longDescription": "Integrate with Evernote to manage notes, notebooks, and tags. Create, read, update, copy, search, and delete notes. Create and list notebooks and tags.", - "bgColor": "#FFFFFF", - "iconName": "EvernoteIcon", - "docsUrl": "https://docs.sim.ai/integrations/evernote", - "operations": [ - { - "name": "Create Note", - "description": "Create a new note in Evernote" - }, - { - "name": "Get Note", - "description": "Retrieve a note from Evernote by its GUID" - }, - { - "name": "Update Note", - "description": "Update an existing note in Evernote" - }, - { - "name": "Delete Note", - "description": "Move a note to the trash in Evernote" - }, - { - "name": "Copy Note", - "description": "Copy a note to another notebook in Evernote" - }, - { - "name": "Search Notes", - "description": "Search for notes in Evernote using the Evernote search grammar" - }, - { - "name": "Get Notebook", - "description": "Retrieve a notebook from Evernote by its GUID" - }, - { - "name": "Create Notebook", - "description": "Create a new notebook in Evernote" - }, - { - "name": "List Notebooks", - "description": "List all notebooks in an Evernote account" - }, - { - "name": "Create Tag", - "description": "Create a new tag in Evernote" - }, - { - "name": "List Tags", - "description": "List all tags in an Evernote account" - } - ], - "operationCount": 11, - "triggers": [], - "triggerCount": 0, - "authType": "api-key", - "category": "tools", - "integrationType": "documents", - "tags": ["note-taking", "knowledge-base"] - }, { "type": "exa", "slug": "exa", diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 98fc82ddbf5..ec70352dcd8 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -4,9 +4,11 @@ import { secureFetchWithValidation, } from '@/lib/core/security/input-validation.server' import { + attachRetryHeaders, type HTTPError, isRetryableError, type RetryOptions, + resolveRetryDelayMs, retryWithExponentialBackoff, } from '@/lib/knowledge/documents/utils' @@ -45,22 +47,26 @@ export async function secureFetchWithRetry( 'url' ) - if (!response.ok && isRetryableError({ status: response.status })) { + /** + * Headers are passed to `isRetryableError` so a rate-limit 403 is + * distinguishable from an authorization denial, and are carried onto the + * thrown error because `retryWithExponentialBackoff` re-evaluates the retry + * condition against it. `resolveRetryDelayMs` prefers `Retry-After` and + * falls back to the epoch-seconds reset header that X (and GitHub's primary + * limit) use instead. + */ + if (!response.ok && isRetryableError({ status: response.status, headers: response.headers })) { const errorText = await response.text() const error: HTTPError = new Error( `HTTP ${response.status}: ${response.statusText} - ${errorText}` ) error.status = response.status error.statusText = response.statusText + attachRetryHeaders(error, response.headers) - const retryAfter = response.headers.get('retry-after') - if (retryAfter) { - const waitMs = Number.isNaN(Number(retryAfter)) - ? Math.max(0, new Date(retryAfter).getTime() - Date.now()) - : Number(retryAfter) * 1000 - if (waitMs > 0) { - error.retryAfterMs = waitMs - } + const waitMs = resolveRetryDelayMs(response.headers) + if (waitMs !== undefined) { + error.retryAfterMs = waitMs } throw error diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index afa2bf1483b..df58cebac26 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockSecureFetchWithValidation } = vi.hoisted(() => ({ mockSecureFetchWithValidation: vi.fn(), @@ -12,7 +12,18 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({ })) import { secureFetchWithRetry } from './secure-fetch.server' -import { isRetryableError } from './utils' +import { + fetchWithRetry, + type HTTPError, + hasRateLimitEvidence, + isRetryableError, + resolveRetryDelayMs, +} from './utils' + +/** Case-insensitive header reader over a plain lowercase-keyed record. */ +function headers(entries: Record) { + return { get: (name: string) => entries[name.toLowerCase()] ?? null } +} /** Builds a minimal SecureFetchResponse-shaped object for tests. */ function fakeResponse( @@ -71,6 +82,20 @@ describe('isRetryableError', () => { it.concurrent('returns true for plain object with status 504', () => { expect(isRetryableError({ status: 504 })).toBe(true) }) + + /** Notion's `service_overload`, which its docs say to "retry the same way as a 429". */ + it.concurrent('returns true for 529 on Error with status', () => { + const error = Object.assign(new Error('service_overload'), { status: 529 }) + expect(isRetryableError(error)).toBe(true) + }) + + it.concurrent('returns true for plain object with status 529', () => { + expect(isRetryableError({ status: 529 })).toBe(true) + }) + + it.concurrent('does not retry 500, which stays adjacent to 529 in the status list', () => { + expect(isRetryableError({ status: 500 })).toBe(false) + }) }) describe('non-retryable status codes', () => { @@ -187,6 +212,268 @@ describe('isRetryableError', () => { expect(isRetryableError(new Error('url resolves to a blocked IP address'))).toBe(false) }) }) + + /** + * GitHub answers both its primary and secondary rate limit with "a `403` or + * `429` response", so a 403 carrying rate-limit headers must retry while a + * bare authorization 403 must not. + */ + describe('rate-limit 403', () => { + it.concurrent('retries a 403 whose x-ratelimit-remaining is 0 (GitHub primary)', () => { + expect( + isRetryableError({ + status: 403, + headers: headers({ 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': '99999999999' }), + }) + ).toBe(true) + }) + + it.concurrent('retries a 403 carrying retry-after (GitHub secondary)', () => { + expect(isRetryableError({ status: 403, headers: headers({ 'retry-after': '60' }) })).toBe( + true + ) + }) + + it.concurrent('retries a 403 whose x-rate-limit-remaining is 0 (X spelling)', () => { + expect( + isRetryableError({ status: 403, headers: headers({ 'x-rate-limit-remaining': '0' }) }) + ).toBe(true) + }) + + it.concurrent('does NOT retry a 403 with no headers at all', () => { + expect(isRetryableError({ status: 403 })).toBe(false) + }) + + it.concurrent('does NOT retry a 403 whose quota is not exhausted', () => { + expect( + isRetryableError({ + status: 403, + headers: headers({ 'x-ratelimit-remaining': '4821', 'x-ratelimit-limit': '5000' }), + }) + ).toBe(false) + }) + + it.concurrent('does NOT retry an authorization 403 on an Error carrying headers', () => { + const error = Object.assign(new Error('Resource not accessible by integration'), { + status: 403, + headers: headers({ 'x-ratelimit-remaining': '4999' }), + }) + expect(isRetryableError(error)).toBe(false) + }) + + it.concurrent('does NOT retry a 401 even with an exhausted quota header', () => { + expect( + isRetryableError({ status: 401, headers: headers({ 'x-ratelimit-remaining': '0' }) }) + ).toBe(false) + }) + }) +}) + +describe('hasRateLimitEvidence', () => { + it.concurrent('returns false for undefined headers', () => { + expect(hasRateLimitEvidence(undefined)).toBe(false) + }) + + it.concurrent('returns false when no rate-limit headers are present', () => { + expect(hasRateLimitEvidence(headers({ 'content-type': 'application/json' }))).toBe(false) + }) + + it.concurrent('returns true on retry-after alone', () => { + expect(hasRateLimitEvidence(headers({ 'retry-after': '30' }))).toBe(true) + }) +}) + +describe('resolveRetryDelayMs', () => { + const NOW = 1_700_000_000_000 + + it.concurrent('prefers Retry-After seconds over the reset header', () => { + const delay = resolveRetryDelayMs( + headers({ 'retry-after': '30', 'x-ratelimit-reset': String(NOW / 1000 + 3600) }), + NOW + ) + expect(delay).toBe(30_000) + }) + + it.concurrent('parses a Retry-After HTTP-date', () => { + const delay = resolveRetryDelayMs( + headers({ 'retry-after': new Date(Date.now() + 60_000).toUTCString() }) + ) + expect(delay).toBeGreaterThan(50_000) + expect(delay).toBeLessThanOrEqual(60_000) + }) + + /** GitHub: "The time at which the current rate limit window resets, in UTC epoch seconds". */ + it.concurrent('falls back to x-ratelimit-reset (GitHub) as epoch seconds', () => { + const delay = resolveRetryDelayMs( + headers({ 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': String(NOW / 1000 + 900) }), + NOW + ) + expect(delay).toBe(900_000) + }) + + /** X sends no Retry-After — x-rate-limit-reset is the only recovery signal. */ + it.concurrent('falls back to x-rate-limit-reset (X spelling) as epoch seconds', () => { + const delay = resolveRetryDelayMs( + headers({ 'x-rate-limit-remaining': '0', 'x-rate-limit-reset': String(NOW / 1000 + 900) }), + NOW + ) + expect(delay).toBe(900_000) + }) + + /** + * GitHub and X stamp their rate-limit headers on every response. Without the + * evidence gate a transient 502 would be handed the rest of the hourly window + * as its wait, which the retry loop clamps to a flat maxDelayMs on every + * attempt — replacing the exponential ladder with 5x the wall-clock stall. + */ + it.concurrent('ignores a reset header when the quota is NOT exhausted', () => { + expect( + resolveRetryDelayMs( + headers({ + 'x-ratelimit-limit': '5000', + 'x-ratelimit-remaining': '4821', + 'x-ratelimit-reset': String(NOW / 1000 + 2400), + }), + NOW + ) + ).toBeUndefined() + }) + + it.concurrent('returns undefined when no usable header is present', () => { + expect(resolveRetryDelayMs(headers({}), NOW)).toBeUndefined() + expect(resolveRetryDelayMs(undefined, NOW)).toBeUndefined() + }) + + it.concurrent('ignores a reset instant already in the past', () => { + expect( + resolveRetryDelayMs( + headers({ 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': String(NOW / 1000 - 60) }), + NOW + ) + ).toBeUndefined() + }) + + it.concurrent('ignores a non-numeric or absurd reset value', () => { + expect( + resolveRetryDelayMs( + headers({ 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': 'soon' }), + NOW + ) + ).toBeUndefined() + // A millisecond value mistaken for epoch seconds would be ~54,000 years out. + expect( + resolveRetryDelayMs( + headers({ 'x-ratelimit-remaining': '0', 'x-ratelimit-reset': String(NOW) }), + NOW + ) + ).toBeUndefined() + }) + + it.concurrent('ignores a zero Retry-After and falls through to the reset header', () => { + const delay = resolveRetryDelayMs( + headers({ 'retry-after': '0', 'x-ratelimit-reset': String(NOW / 1000 + 120) }), + NOW + ) + expect(delay).toBe(120_000) + }) + + /** The 30s default cap in `parseRetryAfter` must not truncate the value here. */ + it.concurrent('does not truncate a long Retry-After — the retry loop owns the cap', () => { + expect(resolveRetryDelayMs(headers({ 'retry-after': '900' }), NOW)).toBe(900_000) + }) +}) + +describe('fetchWithRetry rate-limit handling', () => { + const originalFetch = globalThis.fetch + + afterEach(() => { + globalThis.fetch = originalFetch + }) + + /** Builds a Response-shaped object with real case-insensitive Headers. */ + function response(status: number, headerEntries: Record = {}) { + return { + ok: status >= 200 && status < 300, + status, + statusText: `status-${status}`, + headers: new Headers(headerEntries), + text: async () => 'body', + } as unknown as Response + } + + it('retries a rate-limit 403 (x-ratelimit-remaining: 0) and succeeds', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + response(403, { + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': String(Math.floor(Date.now() / 1000) + 1), + }) + ) + .mockResolvedValueOnce(response(200)) + globalThis.fetch = fetchMock + + const result = await fetchWithRetry('https://api.github.com/repos', {}, FAST_RETRY) + + expect(result.status).toBe(200) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('does not retry an authorization 403 with quota remaining', async () => { + const fetchMock = vi.fn().mockResolvedValue(response(403, { 'x-ratelimit-remaining': '4999' })) + globalThis.fetch = fetchMock + + const result = await fetchWithRetry('https://api.github.com/repos', {}, FAST_RETRY) + + expect(result.status).toBe(403) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('derives the wait from x-rate-limit-reset on a 429 with no Retry-After (X)', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + response(429, { + 'x-rate-limit-remaining': '0', + 'x-rate-limit-reset': String(Math.floor(Date.now() / 1000) + 900), + }) + ) + .mockResolvedValueOnce(response(200)) + globalThis.fetch = fetchMock + + const started = Date.now() + // maxDelayMs clamps the 15-minute window down to 2ms for this test. + const result = await fetchWithRetry('https://api.twitter.com/2/users', {}, FAST_RETRY) + + expect(result.status).toBe(200) + expect(fetchMock).toHaveBeenCalledTimes(2) + // Clamped by maxDelayMs, so the reset window never stalls the loop. + expect(Date.now() - started).toBeLessThan(1000) + }) + + /** + * `@sim/logger` copies an error's own *enumerable* properties into its + * formatted output, and the retry loop logs `{ error }` on every failed + * attempt. `SecureFetchHeaders` keeps its `Set-Cookie` values in an ordinary + * array field, so an enumerable `headers` prints the upstream response's + * cookies. The retry path reads it via `in`, which sees it either way. + */ + it('carries headers on the thrown error without exposing them to the logger', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(response(403, { 'retry-after': '0', 'x-ratelimit-remaining': '0' })) + globalThis.fetch = fetchMock + + const error = await fetchWithRetry('https://api.github.com/repos', {}, FAST_RETRY).then( + () => undefined, + (e) => e as HTTPError + ) + + // Retried, so the headers really did reach the retry condition. + expect(fetchMock).toHaveBeenCalledTimes(FAST_RETRY.maxRetries + 1) + expect(error?.headers?.get('x-ratelimit-remaining')).toBe('0') + expect(Object.keys(error as object)).not.toContain('headers') + }) }) describe('secureFetchWithRetry', () => { @@ -263,6 +550,48 @@ describe('secureFetchWithRetry', () => { expect(options).toMatchObject({ allowHttp: true, timeout: 5000, maxResponseBytes: 1024 }) }) + /** + * Covers `error.headers = response.headers` on the secure path: the retry loop + * re-evaluates the condition against the thrown error, so without the headers + * travelling with it a rate-limit 403 throws on the second evaluation. + */ + it('retries a rate-limit 403 through the secure path and succeeds', async () => { + mockSecureFetchWithValidation + .mockResolvedValueOnce( + fakeResponse(403, { + headers: { + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset': String(Math.floor(Date.now() / 1000) + 1), + }, + }) + ) + .mockResolvedValueOnce(fakeResponse(200)) + + const response = await secureFetchWithRetry( + 'https://api.github.com/repos', + { method: 'GET' }, + FAST_RETRY + ) + + expect(response.status).toBe(200) + expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(2) + }) + + it('does not retry an authorization 403 through the secure path', async () => { + mockSecureFetchWithValidation.mockResolvedValue( + fakeResponse(403, { headers: { 'x-ratelimit-remaining': '4999' } }) + ) + + const response = await secureFetchWithRetry( + 'https://api.github.com/repos', + { method: 'GET' }, + FAST_RETRY + ) + + expect(response.status).toBe(403) + expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1) + }) + it('honors Retry-After (seconds) on a 429 before retrying', async () => { mockSecureFetchWithValidation .mockResolvedValueOnce(fakeResponse(429, { headers: { 'retry-after': '0' } })) diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index 89f30c7db99..f3ed4981ac9 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -2,16 +2,34 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { randomFloat } from '@sim/utils/random' +import { parseRetryAfter } from '@sim/utils/retry' const logger = createLogger('RetryUtils') +/** + * Minimal case-insensitive header reader. Satisfied by the DOM `Headers` class + * and by the header bag on `SecureFetchResponse`, so retry evidence can be read + * without depending on either concrete response type. + */ +export interface HeaderReader { + get(name: string): string | null | undefined +} + export interface HTTPError extends Error { status?: number statusText?: string retryAfterMs?: number + /** + * Response headers carried onto the error so the retry loop can re-evaluate + * rate-limit evidence (`isRetryableError` runs again on the thrown error). + */ + headers?: HeaderReader } -type RetryableError = HTTPError | Error | { status?: number; message?: string } +type RetryableError = + | HTTPError + | Error + | { status?: number; message?: string; headers?: HeaderReader } export interface RetryOptions { maxRetries?: number @@ -41,20 +59,164 @@ function isRetryableErrorType(error: unknown): error is RetryableError { return false } +/** + * Header names carrying the remaining-request count. GitHub spells it + * `x-ratelimit-remaining`; X spells it `x-rate-limit-remaining`. + */ +const RATE_LIMIT_REMAINING_HEADERS = ['x-ratelimit-remaining', 'x-rate-limit-remaining'] as const + +/** + * Header names carrying the window-reset instant as UTC epoch seconds. GitHub + * documents `x-ratelimit-reset` ("The time at which the current rate limit + * window resets, in UTC epoch seconds"); X documents `x-rate-limit-reset` as a + * Unix timestamp. The two spellings differ — both must be read. + */ +const RATE_LIMIT_RESET_HEADERS = ['x-ratelimit-reset', 'x-rate-limit-reset'] as const + +/** + * Upper bound on a reset-derived wait before the value is treated as bogus. + * X documents windows of "15 minutes or 24 hours" and GitHub's primary window + * is an hour, so anything past a day means the header was a delta, a + * millisecond value, or otherwise not the documented epoch-seconds instant. + */ +const MAX_RATE_LIMIT_RESET_WINDOW_MS = 24 * 60 * 60 * 1000 + +function readHeaders(error: RetryableError): HeaderReader | undefined { + if (typeof error !== 'object' || error === null || !('headers' in error)) return undefined + const headers = (error as { headers?: unknown }).headers + if (headers && typeof (headers as HeaderReader).get === 'function') { + return headers as HeaderReader + } + return undefined +} + +/** + * Attaches response headers to a thrown error as a non-enumerable property, so + * the retry loop can re-evaluate rate-limit evidence without the header bag + * reaching a log line. + * + * `@sim/logger` copies an error's own *enumerable* properties into the formatted + * output, and `SecureFetchHeaders` keeps its `Set-Cookie` values in an ordinary + * array field, so a plain assignment prints the upstream response's cookies. + * `readHeaders` uses `in`, which sees non-enumerable properties, so the retry + * path is unaffected. + */ +export function attachRetryHeaders(error: HTTPError, headers: HeaderReader): void { + Object.defineProperty(error, 'headers', { + value: headers, + enumerable: false, + writable: true, + configurable: true, + }) +} + +/** + * True when response headers positively identify a rate-limit rejection rather + * than an authorization denial. + * + * GitHub returns "a `403` or `429` response" for both its primary and its + * secondary rate limit, and directs clients to retry on exactly this evidence: + * "If the `retry-after` response header is present, you should not retry your + * request until after that many seconds has elapsed" and "If the + * `x-ratelimit-remaining` header is `0`, you should not make another request + * until after the time specified by the `x-ratelimit-reset` header." + * + * A bare 403 stays non-retryable — without one of these headers a 403 is an + * ordinary authorization failure and retrying it is pointless. + */ +export function hasRateLimitEvidence(headers: HeaderReader | undefined): boolean { + if (!headers) return false + if (headers.get('retry-after')) return true + return RATE_LIMIT_REMAINING_HEADERS.some((name) => headers.get(name) === '0') +} + +function parseRateLimitResetMs(value: string, nowMs: number): number | undefined { + const resetEpochSeconds = Number(value) + if (!Number.isFinite(resetEpochSeconds) || resetEpochSeconds <= 0) return undefined + const waitMs = resetEpochSeconds * 1000 - nowMs + if (waitMs <= 0 || waitMs > MAX_RATE_LIMIT_RESET_WINDOW_MS) return undefined + return waitMs +} + +/** + * Resolves how long a rate-limited caller must wait, in ms, from response + * headers. Prefers `Retry-After` (seconds or HTTP-date), then falls back to the + * epoch-seconds reset header. + * + * The fallback exists because X does not send `Retry-After`: its documented + * rate-limit headers are `x-rate-limit-limit`, `x-rate-limit-remaining`, and + * `x-rate-limit-reset` (a Unix timestamp) only. GitHub sends `retry-after` on + * secondary limits but signals the primary limit through `x-ratelimit-reset` + * alone. + * + * The reset fallback applies only once {@link hasRateLimitEvidence} holds. + * GitHub and X stamp their rate-limit headers on *every* response, so an + * ungated fallback would turn a transient 502 — quota untouched — into a wait + * until the end of the hourly window, which the retry loop then clamps to a + * flat `maxDelayMs` on every attempt instead of climbing the backoff ladder. + * Gating also matches GitHub's own instruction: "If the `x-ratelimit-remaining` + * header is `0`, you should not make another request until after the time + * specified by the `x-ratelimit-reset` header." + * + * Returns undefined when no header yields a usable future instant, leaving the + * caller on exponential backoff. + */ +export function resolveRetryDelayMs( + headers: HeaderReader | undefined, + nowMs: number = Date.now() +): number | undefined { + if (!headers || !hasRateLimitEvidence(headers)) return undefined + + // Uncapped here on purpose: `retryWithExponentialBackoff` owns the clamp to + // its own maxDelayMs, and the default 30s cap would silently truncate it. + const retryAfterMs = parseRetryAfter(headers.get('retry-after') ?? null, Number.POSITIVE_INFINITY) + if (retryAfterMs !== null && retryAfterMs > 0) return retryAfterMs + + for (const name of RATE_LIMIT_RESET_HEADERS) { + const reset = headers.get(name) + if (!reset) continue + const waitMs = parseRateLimitResetMs(reset, nowMs) + if (waitMs !== undefined) return waitMs + } + + return undefined +} + /** * Default retry condition for rate limiting errors */ export function isRetryableError(error: unknown): boolean { if (!isRetryableErrorType(error)) return false - // Check for rate limiting status codes + /** + * Retryable status codes. 529 is not an IANA-registered status, but Notion + * documents it as `service_overload` — "Notion is temporarily overloaded. + * Respect the `Retry-After` response header and try again later" — and says + * to "retry it the same way as a 429". Without it every Notion call fails + * hard the moment their API sheds load. + */ if ( hasStatus(error) && - (error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504) + (error.status === 429 || + error.status === 502 || + error.status === 503 || + error.status === 504 || + error.status === 529) ) { return true } + /** + * A 403 is retryable only with positive rate-limit evidence in the response + * headers. GitHub answers both its primary and secondary rate limits with + * "a `403` or `429` response", so a rate-limit 403 would otherwise be treated + * as a hard auth failure. Retrying 403 unconditionally would be wrong — 403 + * normally means authorization denied. + */ + if (hasStatus(error) && error.status === 403 && hasRateLimitEvidence(readHeaders(error))) { + return true + } + // Check for network-level errors (DNS, connection, timeout) const errorMessage = toError(error).message const lowerMessage = errorMessage.toLowerCase() @@ -136,14 +298,24 @@ export async function retryWithExponentialBackoff( throw lastError } - // Use Retry-After if the server told us how long to wait, otherwise exponential backoff. - // Cap Retry-After at maxDelayMs to bound total retry duration (matches Google Cloud SDK behavior). + /** + * Use the server-stated wait (Retry-After, or the rate-limit reset + * header) when present, otherwise exponential backoff. The wait is capped + * at maxDelayMs to bound total retry duration. + * + * Note the tradeoff the cap creates for long rate-limit windows: GitHub's + * primary window is an hour and X's is 15 minutes, both far beyond the + * 30s default, so every retry fires before the window reopens and the + * attempts are spent for nothing. Raising the cap would instead stall a + * sync for the full window. Neither provider documents a bound here, so + * the existing conservative cap stands and the mismatch is logged. + */ const retryAfterMs = (lastError as HTTPError)?.retryAfterMs const cappedRetryAfter = retryAfterMs ? Math.min(retryAfterMs, maxDelayMs) : undefined if (retryAfterMs && retryAfterMs > maxDelayMs) { logger.warn( - `Retry-After ${retryAfterMs}ms exceeds maxDelayMs ${maxDelayMs}ms — capping to ${maxDelayMs}ms` + `Server-stated retry wait ${retryAfterMs}ms exceeds maxDelayMs ${maxDelayMs}ms — capping to ${maxDelayMs}ms; retries will fire before the rate-limit window reopens` ) } @@ -151,7 +323,7 @@ export async function retryWithExponentialBackoff( const actualDelay = cappedRetryAfter ?? Math.min(delay + jitter, maxDelayMs) logger.info( - `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${cappedRetryAfter ? ' (Retry-After)' : ''}` + `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${cappedRetryAfter ? ' (server-stated)' : ''}` ) await sleep(actualDelay) @@ -187,24 +359,26 @@ export async function fetchWithRetry( return retryWithExponentialBackoff(async () => { const response = await fetch(url, options) - // If response is not ok and status indicates rate limiting, throw an error - if (!response.ok && isRetryableError({ status: response.status })) { + // If response is not ok and status indicates rate limiting, throw an error. + // Headers are part of the evidence: a 403 is retryable only when they prove + // a rate limit rather than an authorization denial. + if (!response.ok && isRetryableError({ status: response.status, headers: response.headers })) { const errorText = await response.text() const error: HTTPError = new Error( `HTTP ${response.status}: ${response.statusText} - ${errorText}` ) error.status = response.status error.statusText = response.statusText - - // Pass Retry-After to the retry loop so it replaces exponential backoff - const retryAfter = response.headers.get('Retry-After') - if (retryAfter) { - const waitMs = Number.isNaN(Number(retryAfter)) - ? Math.max(0, new Date(retryAfter).getTime() - Date.now()) - : Number(retryAfter) * 1000 - if (waitMs > 0) { - error.retryAfterMs = waitMs - } + // The retry loop re-runs the retry condition against this error, so the + // headers must travel with it or a rate-limit 403 would throw immediately. + attachRetryHeaders(error, response.headers) + + // Pass the server-stated wait to the retry loop so it replaces exponential + // backoff. Falls back to the epoch-seconds reset header when the provider + // sends no Retry-After (X never does). + const waitMs = resolveRetryDelayMs(response.headers) + if (waitMs !== undefined) { + error.retryAfterMs = waitMs } throw error diff --git a/apps/sim/lib/oauth/docusign.ts b/apps/sim/lib/oauth/docusign.ts new file mode 100644 index 00000000000..27d038b86d5 --- /dev/null +++ b/apps/sim/lib/oauth/docusign.ts @@ -0,0 +1,47 @@ +import { env } from '@/lib/core/config/env' + +/** + * DocuSign developer/demo and production are separate account systems with + * separate authentication services, so a token issued by one is not accepted by + * the other. Going live means switching hosts: "The authentication service host + * name needs to be changed from https://account-d.docusign.com to + * https://account.docusign.com in your app's OAuth configuration" + * (docusign.com/blog/developers — configuring your production account). + * + * Default kept on demo so existing installations keep their current behavior. + */ +const DOCUSIGN_DEMO_AUTH_HOST = 'account-d.docusign.com' +const DOCUSIGN_PRODUCTION_AUTH_HOST = 'account.docusign.com' + +/** + * Resolves the DocuSign OAuth host from `DOCUSIGN_AUTH_HOST`. Accepts a bare host + * or a full origin; anything that is not one of the two DocuSign authentication + * hosts falls back to the demo host, so a typo can never redirect the OAuth flow + * (and the bearer token that follows) to an attacker-controlled origin. + */ +function getDocusignAuthHost(): string { + const configured = env.DOCUSIGN_AUTH_HOST?.trim() + if (!configured) return DOCUSIGN_DEMO_AUTH_HOST + const host = configured + .replace(/^https?:\/\//, '') + .replace(/\/.*$/, '') + .toLowerCase() + return host === DOCUSIGN_PRODUCTION_AUTH_HOST || host === DOCUSIGN_DEMO_AUTH_HOST + ? host + : DOCUSIGN_DEMO_AUTH_HOST +} + +/** Absolute DocuSign OAuth endpoint (e.g. `/oauth/token`) on the configured host. */ +export function getDocusignOAuthUrl(path: string): string { + return `https://${getDocusignAuthHost()}${path}` +} + +/** + * DocuSign web-app base for envelope deep links, kept in lockstep with + * {@link getDocusignAuthHost}: demo envelopes only exist in `appdemo.docusign.com`. + */ +export function getDocusignWebBase(): string { + return getDocusignAuthHost() === DOCUSIGN_PRODUCTION_AUTH_HOST + ? 'https://app.docusign.com' + : 'https://appdemo.docusign.com' +} diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 41054c655c5..b06cd09aaf0 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -65,6 +65,7 @@ afterAll(resetEnvMock) import { DEFAULT_MAX_ERROR_BODY_BYTES } from '@/lib/core/utils/stream-limits' import { refreshOAuthToken } from '@/lib/oauth' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' /** * Default OAuth token response for successful requests. @@ -382,9 +383,12 @@ describe('OAuth Token Refresh', () => { string, { headers: Record; body: string }, ] - expect(requestOptions.headers['User-Agent']).toBe( - 'sim-studio/1.0 (https://github.com/simstudioai/sim)' - ) + expect(requestOptions.headers['User-Agent']).toBe(REDDIT_USER_AGENT) + /** + * Reddit rate-limits generic User-Agents, so the shared constant must keep + * the documented `::` shape wherever it is used. + */ + expect(REDDIT_USER_AGENT).toMatch(/^[a-z]+:[\w.-]+:v[\d.]+ \(.+\)$/) }) }) diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index e8f43754cf9..b11c567a2b7 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -71,12 +71,14 @@ import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' import { SALESFORCE_ADDITIONAL_PROVIDER_IDS, SALESFORCE_LOGIN_HOSTS, SALESFORCE_PROVIDER_ID_LABELS, } from '@/lib/oauth/salesforce' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' import type { OAuthProviderConfig } from './types' const logger = createLogger('OAuth') @@ -1524,6 +1526,12 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { clientId, clientSecret, useBasicAuth: false, + // Box refresh tokens are single-use: "the Refresh Token is invalidated and a + // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and + // can be used to obtain a new Access Token and Refresh Token only once." + // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the + // new token is discarded and the credential dies on the second refresh. + supportsRefreshTokenRotation: true, } } case 'docusign': { @@ -1533,7 +1541,7 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { 'DOCUSIGN_CLIENT_SECRET' ) return { - tokenEndpoint: 'https://account-d.docusign.com/oauth/token', + tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), clientId, clientSecret, useBasicAuth: true, @@ -1580,7 +1588,7 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { clientSecret, useBasicAuth: true, additionalHeaders: { - 'User-Agent': 'sim-studio/1.0 (https://github.com/simstudioai/sim)', + 'User-Agent': REDDIT_USER_AGENT, }, } } @@ -1779,9 +1787,16 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { case 'zoho-desk': { // Zoho's refresh_token grant returns a new access token but no new refresh // token, so rotation stays off (the existing refresh token is preserved). - // The refresh must target the accounts server; a US/multi-DC-enabled client - // uses accounts.zoho.com. Data residency for API calls is honored separately - // via the persisted Desk base URL derived from the token response api_domain. + // The refresh must target the accounts server of the data center that issued + // the token - "if location=eu, you will need to make access token request to + // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). + // accounts.zoho.com is correct here because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts + // server, so every refresh token in the system is US-issued. Making refresh + // DC-aware requires making the grant DC-aware first (read the `accounts-server` + // callback param) and threading the credential's persisted `__zoho_domain__` + // marker into refreshOAuthToken, which today only receives the token string. + // Data residency for API calls is already honored via that persisted Desk base. const { clientId, clientSecret } = getConfiguredClientCredentials( 'zoho-desk', 'ZOHO_CLIENT_ID', diff --git a/apps/sim/lib/oauth/utils.test.ts b/apps/sim/lib/oauth/utils.test.ts index 656938fae49..2193efebac1 100644 --- a/apps/sim/lib/oauth/utils.test.ts +++ b/apps/sim/lib/oauth/utils.test.ts @@ -698,6 +698,34 @@ describe('getMissingRequiredScopes', () => { expect(missing).toEqual([]) }) + it.concurrent('accepts calendar for a required calendar.readonly via the generic rule', () => { + const credential = { scopes: ['https://www.googleapis.com/auth/calendar'] } + const missing = getMissingRequiredScopes(credential, [ + 'https://www.googleapis.com/auth/calendar.readonly', + ]) + + expect(missing).toEqual([]) + }) + + /** + * The rule derives only the bare read-write scope. Sim requests `gmail.send`, + * `gmail.modify` and `gmail.labels` but never `.../auth/gmail`, so a consumer + * must require one of the scopes actually granted rather than `gmail.readonly`. + */ + it.concurrent('does not treat unrelated gmail scopes as covering gmail.readonly', () => { + const credential = { + scopes: [ + 'https://www.googleapis.com/auth/gmail.send', + 'https://www.googleapis.com/auth/gmail.labels', + ], + } + const missing = getMissingRequiredScopes(credential, [ + 'https://www.googleapis.com/auth/gmail.readonly', + ]) + + expect(missing).toEqual(['https://www.googleapis.com/auth/gmail.readonly']) + }) + it.concurrent('should ignore offline.access in required scopes', () => { const credential = { scopes: ['read'] } const missing = getMissingRequiredScopes(credential, ['read', 'offline.access']) diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index 055a45b3586..616a4a1863b 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -23,9 +23,11 @@ export const SCOPE_DESCRIPTIONS: Record = { // Google scopes 'https://www.googleapis.com/auth/gmail.send': 'Send emails', 'https://www.googleapis.com/auth/gmail.labels': 'View and manage email labels', + 'https://www.googleapis.com/auth/gmail.readonly': 'View email messages and settings', 'https://www.googleapis.com/auth/gmail.modify': 'View and manage email messages', 'https://www.googleapis.com/auth/drive.file': 'View and manage Google Drive files', 'https://www.googleapis.com/auth/drive': 'Access all Google Drive files', + 'https://www.googleapis.com/auth/calendar.readonly': 'View calendars and events', 'https://www.googleapis.com/auth/calendar': 'View and manage calendar', 'https://www.googleapis.com/auth/contacts': 'View and manage Google Contacts', 'https://www.googleapis.com/auth/tasks': 'Create, read, update, and delete Google Tasks', @@ -710,6 +712,10 @@ export function getMissingRequiredScopes( * `.../auth/ediscovery.readonly`. Without this, narrowing a consumer to the * least-privileged scope would report every already-connected credential as * missing it and prompt a re-consent that grants nothing new. + * + * This only derives a scope Sim actually requests. A consumer must never + * require a scope absent from its provider's `scopes` array — no credential can + * carry it, since that array is what the authorize request asks for. */ function isScopeSatisfiedBy(required: string, granted: ReadonlySet): boolean { const readonlySuffix = '.readonly' diff --git a/apps/sim/lib/webhooks/providers/monday.ts b/apps/sim/lib/webhooks/providers/monday.ts index b0ad31267d2..64ba5bde7f4 100644 --- a/apps/sim/lib/webhooks/providers/monday.ts +++ b/apps/sim/lib/webhooks/providers/monday.ts @@ -16,11 +16,10 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' const logger = createLogger('WebhookProvider:Monday') -const MONDAY_API_URL = 'https://api.monday.com/v2' - /** * Resolves an OAuth access token from the webhook's credential configuration. * Follows the Airtable pattern: credentialId → getCredentialOwner → refreshAccessTokenIfNeeded. @@ -109,11 +108,7 @@ export const mondayHandler: WebhookProviderHandler = { try { const response = await fetch(MONDAY_API_URL, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'API-Version': '2024-10', - Authorization: accessToken, - }, + headers: mondayHeaders(accessToken), body: JSON.stringify({ query: `mutation { create_webhook(board_id: ${boardIdValidation.sanitized}, url: ${JSON.stringify(notificationUrl)}, event: ${eventType}) { id board_id } }`, }), @@ -226,11 +221,7 @@ export const mondayHandler: WebhookProviderHandler = { try { const response = await fetch(MONDAY_API_URL, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'API-Version': '2024-10', - Authorization: accessToken, - }, + headers: mondayHeaders(accessToken), body: JSON.stringify({ query: `mutation { delete_webhook(id: ${externalIdValidation.sanitized}) { id board_id } }`, }), diff --git a/apps/sim/tools/evernote/copy_note.ts b/apps/sim/tools/evernote/copy_note.ts deleted file mode 100644 index 9493e6c9b98..00000000000 --- a/apps/sim/tools/evernote/copy_note.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteCopyNoteParams, EvernoteCopyNoteResponse } from './types' - -export const evernoteCopyNoteTool: ToolConfig = { - id: 'evernote_copy_note', - name: 'Evernote Copy Note', - description: 'Copy a note to another notebook in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - noteGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the note to copy', - }, - toNotebookGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the destination notebook', - }, - }, - - request: { - url: '/api/tools/evernote/copy-note', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - noteGuid: params.noteGuid, - toNotebookGuid: params.toNotebookGuid, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to copy note') - } - return { - success: true, - output: { note: data.output.note }, - } - }, - - outputs: { - note: { - type: 'object', - description: 'The copied note metadata', - properties: { - guid: { type: 'string', description: 'New note GUID' }, - title: { type: 'string', description: 'Note title' }, - notebookGuid: { - type: 'string', - description: 'GUID of the destination notebook', - optional: true, - }, - created: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - updated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/create_note.ts b/apps/sim/tools/evernote/create_note.ts deleted file mode 100644 index 281735f6ac1..00000000000 --- a/apps/sim/tools/evernote/create_note.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteCreateNoteParams, EvernoteCreateNoteResponse } from './types' - -export const evernoteCreateNoteTool: ToolConfig< - EvernoteCreateNoteParams, - EvernoteCreateNoteResponse -> = { - id: 'evernote_create_note', - name: 'Evernote Create Note', - description: 'Create a new note in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - title: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'Title of the note', - }, - content: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'Content of the note (plain text or ENML)', - }, - notebookGuid: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'GUID of the notebook to create the note in (defaults to default notebook)', - }, - tagNames: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'Comma-separated list of tag names to apply', - }, - }, - - request: { - url: '/api/tools/evernote/create-note', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - title: params.title, - content: params.content, - notebookGuid: params.notebookGuid || null, - tagNames: params.tagNames || null, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to create note') - } - return { - success: true, - output: { note: data.output.note }, - } - }, - - outputs: { - note: { - type: 'object', - description: 'The created note', - properties: { - guid: { type: 'string', description: 'Unique identifier of the note' }, - title: { type: 'string', description: 'Title of the note' }, - content: { type: 'string', description: 'ENML content of the note', optional: true }, - notebookGuid: { - type: 'string', - description: 'GUID of the containing notebook', - optional: true, - }, - tagNames: { - type: 'array', - description: 'Tag names applied to the note', - optional: true, - }, - created: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - updated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/create_notebook.ts b/apps/sim/tools/evernote/create_notebook.ts deleted file mode 100644 index ba46e48b50b..00000000000 --- a/apps/sim/tools/evernote/create_notebook.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteCreateNotebookParams, EvernoteCreateNotebookResponse } from './types' - -export const evernoteCreateNotebookTool: ToolConfig< - EvernoteCreateNotebookParams, - EvernoteCreateNotebookResponse -> = { - id: 'evernote_create_notebook', - name: 'Evernote Create Notebook', - description: 'Create a new notebook in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - name: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'Name for the new notebook', - }, - stack: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'Stack name to group the notebook under', - }, - }, - - request: { - url: '/api/tools/evernote/create-notebook', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - name: params.name, - stack: params.stack || null, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to create notebook') - } - return { - success: true, - output: { notebook: data.output.notebook }, - } - }, - - outputs: { - notebook: { - type: 'object', - description: 'The created notebook', - properties: { - guid: { type: 'string', description: 'Notebook GUID' }, - name: { type: 'string', description: 'Notebook name' }, - defaultNotebook: { type: 'boolean', description: 'Whether this is the default notebook' }, - serviceCreated: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - serviceUpdated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - stack: { type: 'string', description: 'Notebook stack name', optional: true }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/create_tag.ts b/apps/sim/tools/evernote/create_tag.ts deleted file mode 100644 index aeaa3d2dbf6..00000000000 --- a/apps/sim/tools/evernote/create_tag.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteCreateTagParams, EvernoteCreateTagResponse } from './types' - -export const evernoteCreateTagTool: ToolConfig = - { - id: 'evernote_create_tag', - name: 'Evernote Create Tag', - description: 'Create a new tag in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - name: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'Name for the new tag', - }, - parentGuid: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'GUID of the parent tag for hierarchy', - }, - }, - - request: { - url: '/api/tools/evernote/create-tag', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - name: params.name, - parentGuid: params.parentGuid || null, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to create tag') - } - return { - success: true, - output: { tag: data.output.tag }, - } - }, - - outputs: { - tag: { - type: 'object', - description: 'The created tag', - properties: { - guid: { type: 'string', description: 'Tag GUID' }, - name: { type: 'string', description: 'Tag name' }, - parentGuid: { type: 'string', description: 'Parent tag GUID', optional: true }, - updateSequenceNum: { - type: 'number', - description: 'Update sequence number', - optional: true, - }, - }, - }, - }, - } diff --git a/apps/sim/tools/evernote/delete_note.ts b/apps/sim/tools/evernote/delete_note.ts deleted file mode 100644 index 6983a78d3f8..00000000000 --- a/apps/sim/tools/evernote/delete_note.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteDeleteNoteParams, EvernoteDeleteNoteResponse } from './types' - -export const evernoteDeleteNoteTool: ToolConfig< - EvernoteDeleteNoteParams, - EvernoteDeleteNoteResponse -> = { - id: 'evernote_delete_note', - name: 'Evernote Delete Note', - description: 'Move a note to the trash in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - noteGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the note to delete', - }, - }, - - request: { - url: '/api/tools/evernote/delete-note', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - noteGuid: params.noteGuid, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to delete note') - } - return { - success: true, - output: { - success: true, - noteGuid: data.output.noteGuid, - }, - } - }, - - outputs: { - success: { - type: 'boolean', - description: 'Whether the note was successfully deleted', - }, - noteGuid: { - type: 'string', - description: 'GUID of the deleted note', - }, - }, -} diff --git a/apps/sim/tools/evernote/get_note.ts b/apps/sim/tools/evernote/get_note.ts deleted file mode 100644 index 4773bd23700..00000000000 --- a/apps/sim/tools/evernote/get_note.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteGetNoteParams, EvernoteGetNoteResponse } from './types' - -export const evernoteGetNoteTool: ToolConfig = { - id: 'evernote_get_note', - name: 'Evernote Get Note', - description: 'Retrieve a note from Evernote by its GUID', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - noteGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the note to retrieve', - }, - withContent: { - type: 'boolean', - required: false, - visibility: 'user-or-llm', - description: 'Whether to include note content (default: true)', - }, - }, - - request: { - url: '/api/tools/evernote/get-note', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - noteGuid: params.noteGuid, - withContent: params.withContent ?? true, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to get note') - } - return { - success: true, - output: { note: data.output.note }, - } - }, - - outputs: { - note: { - type: 'object', - description: 'The retrieved note', - properties: { - guid: { type: 'string', description: 'Unique identifier of the note' }, - title: { type: 'string', description: 'Title of the note' }, - content: { type: 'string', description: 'ENML content of the note', optional: true }, - contentLength: { - type: 'number', - description: 'Length of the note content', - optional: true, - }, - notebookGuid: { - type: 'string', - description: 'GUID of the containing notebook', - optional: true, - }, - tagGuids: { type: 'array', description: 'GUIDs of tags on the note', optional: true }, - tagNames: { type: 'array', description: 'Names of tags on the note', optional: true }, - created: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - updated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - active: { type: 'boolean', description: 'Whether the note is active (not in trash)' }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/get_notebook.ts b/apps/sim/tools/evernote/get_notebook.ts deleted file mode 100644 index 78a2fd59fa6..00000000000 --- a/apps/sim/tools/evernote/get_notebook.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteGetNotebookParams, EvernoteGetNotebookResponse } from './types' - -export const evernoteGetNotebookTool: ToolConfig< - EvernoteGetNotebookParams, - EvernoteGetNotebookResponse -> = { - id: 'evernote_get_notebook', - name: 'Evernote Get Notebook', - description: 'Retrieve a notebook from Evernote by its GUID', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - notebookGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the notebook to retrieve', - }, - }, - - request: { - url: '/api/tools/evernote/get-notebook', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - notebookGuid: params.notebookGuid, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to get notebook') - } - return { - success: true, - output: { notebook: data.output.notebook }, - } - }, - - outputs: { - notebook: { - type: 'object', - description: 'The retrieved notebook', - properties: { - guid: { type: 'string', description: 'Notebook GUID' }, - name: { type: 'string', description: 'Notebook name' }, - defaultNotebook: { type: 'boolean', description: 'Whether this is the default notebook' }, - serviceCreated: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - serviceUpdated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - stack: { type: 'string', description: 'Notebook stack name', optional: true }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/index.ts b/apps/sim/tools/evernote/index.ts deleted file mode 100644 index 08819e0baf4..00000000000 --- a/apps/sim/tools/evernote/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -export { evernoteCopyNoteTool } from './copy_note' -export { evernoteCreateNoteTool } from './create_note' -export { evernoteCreateNotebookTool } from './create_notebook' -export { evernoteCreateTagTool } from './create_tag' -export { evernoteDeleteNoteTool } from './delete_note' -export { evernoteGetNoteTool } from './get_note' -export { evernoteGetNotebookTool } from './get_notebook' -export { evernoteListNotebooksTool } from './list_notebooks' -export { evernoteListTagsTool } from './list_tags' -export { evernoteSearchNotesTool } from './search_notes' -export * from './types' -export { evernoteUpdateNoteTool } from './update_note' diff --git a/apps/sim/tools/evernote/list_notebooks.ts b/apps/sim/tools/evernote/list_notebooks.ts deleted file mode 100644 index b2b9756c7e8..00000000000 --- a/apps/sim/tools/evernote/list_notebooks.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteListNotebooksParams, EvernoteListNotebooksResponse } from './types' - -export const evernoteListNotebooksTool: ToolConfig< - EvernoteListNotebooksParams, - EvernoteListNotebooksResponse -> = { - id: 'evernote_list_notebooks', - name: 'Evernote List Notebooks', - description: 'List all notebooks in an Evernote account', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - }, - - request: { - url: '/api/tools/evernote/list-notebooks', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to list notebooks') - } - return { - success: true, - output: { notebooks: data.output.notebooks }, - } - }, - - outputs: { - notebooks: { - type: 'array', - description: 'List of notebooks', - properties: { - guid: { type: 'string', description: 'Notebook GUID' }, - name: { type: 'string', description: 'Notebook name' }, - defaultNotebook: { type: 'boolean', description: 'Whether this is the default notebook' }, - serviceCreated: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - serviceUpdated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - stack: { type: 'string', description: 'Notebook stack name', optional: true }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/list_tags.ts b/apps/sim/tools/evernote/list_tags.ts deleted file mode 100644 index 65cb5a04fdd..00000000000 --- a/apps/sim/tools/evernote/list_tags.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteListTagsParams, EvernoteListTagsResponse } from './types' - -export const evernoteListTagsTool: ToolConfig = { - id: 'evernote_list_tags', - name: 'Evernote List Tags', - description: 'List all tags in an Evernote account', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - }, - - request: { - url: '/api/tools/evernote/list-tags', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to list tags') - } - return { - success: true, - output: { tags: data.output.tags }, - } - }, - - outputs: { - tags: { - type: 'array', - description: 'List of tags', - properties: { - guid: { type: 'string', description: 'Tag GUID' }, - name: { type: 'string', description: 'Tag name' }, - parentGuid: { type: 'string', description: 'Parent tag GUID', optional: true }, - updateSequenceNum: { - type: 'number', - description: 'Update sequence number', - optional: true, - }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/search_notes.ts b/apps/sim/tools/evernote/search_notes.ts deleted file mode 100644 index a75056434d3..00000000000 --- a/apps/sim/tools/evernote/search_notes.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteSearchNotesParams, EvernoteSearchNotesResponse } from './types' - -export const evernoteSearchNotesTool: ToolConfig< - EvernoteSearchNotesParams, - EvernoteSearchNotesResponse -> = { - id: 'evernote_search_notes', - name: 'Evernote Search Notes', - description: 'Search for notes in Evernote using the Evernote search grammar', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - query: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'Search query using Evernote search grammar (e.g., "tag:work intitle:meeting")', - }, - notebookGuid: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'Restrict search to a specific notebook by GUID', - }, - offset: { - type: 'number', - required: false, - visibility: 'user-or-llm', - description: 'Starting index for results (default: 0)', - }, - maxNotes: { - type: 'number', - required: false, - visibility: 'user-or-llm', - description: 'Maximum number of notes to return (default: 25)', - }, - }, - - request: { - url: '/api/tools/evernote/search-notes', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - query: params.query, - notebookGuid: params.notebookGuid || null, - offset: params.offset ?? 0, - maxNotes: params.maxNotes ?? 25, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to search notes') - } - return { - success: true, - output: { - totalNotes: data.output.totalNotes, - notes: data.output.notes, - }, - } - }, - - outputs: { - totalNotes: { - type: 'number', - description: 'Total number of matching notes', - }, - notes: { - type: 'array', - description: 'List of matching note metadata', - properties: { - guid: { type: 'string', description: 'Note GUID' }, - title: { type: 'string', description: 'Note title', optional: true }, - contentLength: { type: 'number', description: 'Content length in bytes', optional: true }, - created: { type: 'number', description: 'Creation timestamp', optional: true }, - updated: { type: 'number', description: 'Last updated timestamp', optional: true }, - notebookGuid: { type: 'string', description: 'Containing notebook GUID', optional: true }, - tagGuids: { type: 'array', description: 'Tag GUIDs', optional: true }, - }, - }, - }, -} diff --git a/apps/sim/tools/evernote/types.ts b/apps/sim/tools/evernote/types.ts deleted file mode 100644 index 92655b5141e..00000000000 --- a/apps/sim/tools/evernote/types.ts +++ /dev/null @@ -1,166 +0,0 @@ -import type { ToolResponse } from '@/tools/types' - -interface EvernoteBaseParams { - apiKey: string -} - -export interface EvernoteCreateNoteParams extends EvernoteBaseParams { - title: string - content: string - notebookGuid?: string - tagNames?: string -} - -export interface EvernoteGetNoteParams extends EvernoteBaseParams { - noteGuid: string - withContent?: boolean -} - -export interface EvernoteUpdateNoteParams extends EvernoteBaseParams { - noteGuid: string - title?: string - content?: string - notebookGuid?: string - tagNames?: string -} - -export interface EvernoteDeleteNoteParams extends EvernoteBaseParams { - noteGuid: string -} - -export interface EvernoteSearchNotesParams extends EvernoteBaseParams { - query: string - notebookGuid?: string - offset?: number - maxNotes?: number -} - -export interface EvernoteListNotebooksParams extends EvernoteBaseParams {} - -export interface EvernoteGetNotebookParams extends EvernoteBaseParams { - notebookGuid: string -} - -export interface EvernoteCreateNotebookParams extends EvernoteBaseParams { - name: string - stack?: string -} - -export interface EvernoteListTagsParams extends EvernoteBaseParams {} - -export interface EvernoteCreateTagParams extends EvernoteBaseParams { - name: string - parentGuid?: string -} - -export interface EvernoteCopyNoteParams extends EvernoteBaseParams { - noteGuid: string - toNotebookGuid: string -} - -interface EvernoteNoteOutput { - guid: string - title: string - content: string | null - contentLength: number | null - created: number | null - updated: number | null - active: boolean - notebookGuid: string | null - tagGuids: string[] - tagNames: string[] -} - -interface EvernoteNotebookOutput { - guid: string - name: string - defaultNotebook: boolean - serviceCreated: number | null - serviceUpdated: number | null - stack: string | null -} - -interface EvernoteNoteMetadataOutput { - guid: string - title: string | null - contentLength: number | null - created: number | null - updated: number | null - notebookGuid: string | null - tagGuids: string[] -} - -interface EvernoteTagOutput { - guid: string - name: string - parentGuid: string | null - updateSequenceNum: number | null -} - -export interface EvernoteCreateNoteResponse extends ToolResponse { - output: { - note: EvernoteNoteOutput - } -} - -export interface EvernoteGetNoteResponse extends ToolResponse { - output: { - note: EvernoteNoteOutput - } -} - -export interface EvernoteUpdateNoteResponse extends ToolResponse { - output: { - note: EvernoteNoteOutput - } -} - -export interface EvernoteDeleteNoteResponse extends ToolResponse { - output: { - success: boolean - noteGuid: string - } -} - -export interface EvernoteSearchNotesResponse extends ToolResponse { - output: { - totalNotes: number - notes: EvernoteNoteMetadataOutput[] - } -} - -export interface EvernoteListNotebooksResponse extends ToolResponse { - output: { - notebooks: EvernoteNotebookOutput[] - } -} - -export interface EvernoteGetNotebookResponse extends ToolResponse { - output: { - notebook: EvernoteNotebookOutput - } -} - -export interface EvernoteCreateNotebookResponse extends ToolResponse { - output: { - notebook: EvernoteNotebookOutput - } -} - -export interface EvernoteListTagsResponse extends ToolResponse { - output: { - tags: EvernoteTagOutput[] - } -} - -export interface EvernoteCreateTagResponse extends ToolResponse { - output: { - tag: EvernoteTagOutput - } -} - -export interface EvernoteCopyNoteResponse extends ToolResponse { - output: { - note: EvernoteNoteOutput - } -} diff --git a/apps/sim/tools/evernote/update_note.ts b/apps/sim/tools/evernote/update_note.ts deleted file mode 100644 index 48872e6c6e4..00000000000 --- a/apps/sim/tools/evernote/update_note.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { ToolConfig } from '@/tools/types' -import type { EvernoteUpdateNoteParams, EvernoteUpdateNoteResponse } from './types' - -export const evernoteUpdateNoteTool: ToolConfig< - EvernoteUpdateNoteParams, - EvernoteUpdateNoteResponse -> = { - id: 'evernote_update_note', - name: 'Evernote Update Note', - description: 'Update an existing note in Evernote', - version: '1.0.0', - - params: { - apiKey: { - type: 'string', - required: true, - visibility: 'user-only', - description: 'Evernote developer token', - }, - noteGuid: { - type: 'string', - required: true, - visibility: 'user-or-llm', - description: 'GUID of the note to update', - }, - title: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'New title for the note', - }, - content: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'New content for the note (plain text or ENML)', - }, - notebookGuid: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'GUID of the notebook to move the note to', - }, - tagNames: { - type: 'string', - required: false, - visibility: 'user-or-llm', - description: 'Comma-separated list of tag names (replaces existing tags)', - }, - }, - - request: { - url: '/api/tools/evernote/update-note', - method: 'POST', - headers: () => ({ 'Content-Type': 'application/json' }), - body: (params) => ({ - apiKey: params.apiKey, - noteGuid: params.noteGuid, - title: params.title || null, - content: params.content || null, - notebookGuid: params.notebookGuid || null, - tagNames: params.tagNames || null, - }), - }, - - transformResponse: async (response) => { - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to update note') - } - return { - success: true, - output: { note: data.output.note }, - } - }, - - outputs: { - note: { - type: 'object', - description: 'The updated note', - properties: { - guid: { type: 'string', description: 'Unique identifier of the note' }, - title: { type: 'string', description: 'Title of the note' }, - content: { type: 'string', description: 'ENML content of the note', optional: true }, - notebookGuid: { - type: 'string', - description: 'GUID of the containing notebook', - optional: true, - }, - tagNames: { type: 'array', description: 'Tag names on the note', optional: true }, - created: { - type: 'number', - description: 'Creation timestamp in milliseconds', - optional: true, - }, - updated: { - type: 'number', - description: 'Last updated timestamp in milliseconds', - optional: true, - }, - }, - }, - }, -} diff --git a/apps/sim/tools/generated/tool-ids.ts b/apps/sim/tools/generated/tool-ids.ts index 75a043152f3..9623d5bdace 100644 --- a/apps/sim/tools/generated/tool-ids.ts +++ b/apps/sim/tools/generated/tool-ids.ts @@ -3,7 +3,7 @@ /** Every registered tool id, including versioned variants. */ const toolIds: string[] = JSON.parse( - '["a2a_cancel_task","a2a_get_agent_card","a2a_get_task","a2a_send_message","agentmail_create_draft","agentmail_create_inbox","agentmail_delete_draft","agentmail_delete_inbox","agentmail_delete_thread","agentmail_forward_message","agentmail_get_draft","agentmail_get_inbox","agentmail_get_message","agentmail_get_thread","agentmail_list_drafts","agentmail_list_inboxes","agentmail_list_messages","agentmail_list_threads","agentmail_reply_message","agentmail_send_draft","agentmail_send_message","agentmail_update_draft","agentmail_update_inbox","agentmail_update_message","agentmail_update_thread","agentphone_create_call","agentphone_create_contact","agentphone_create_number","agentphone_delete_contact","agentphone_get_call","agentphone_get_call_transcript","agentphone_get_contact","agentphone_get_conversation","agentphone_get_conversation_messages","agentphone_get_number_messages","agentphone_get_usage","agentphone_get_usage_daily","agentphone_get_usage_monthly","agentphone_list_calls","agentphone_list_contacts","agentphone_list_conversations","agentphone_list_numbers","agentphone_react_to_message","agentphone_release_number","agentphone_send_message","agentphone_update_contact","agentphone_update_conversation","agiloft_async_status","agiloft_attach_file","agiloft_attachment_info","agiloft_create_record","agiloft_delete_record","agiloft_get_choice_line_id","agiloft_list_tables","agiloft_lock_record","agiloft_nlp_search","agiloft_read_record","agiloft_remove_attachment","agiloft_retrieve_attachment","agiloft_run_action_button","agiloft_saved_search","agiloft_search_records","agiloft_select_records","agiloft_update_record","agiloft_upsert_record","ahrefs_anchors","ahrefs_backlinks","ahrefs_backlinks_stats","ahrefs_batch_analysis","ahrefs_broken_backlinks","ahrefs_domain_rating","ahrefs_domain_rating_history","ahrefs_keyword_overview","ahrefs_keywords_history","ahrefs_metrics","ahrefs_metrics_history","ahrefs_organic_competitors","ahrefs_organic_keywords","ahrefs_paid_pages","ahrefs_rank_tracker_competitors_overview","ahrefs_rank_tracker_competitors_stats","ahrefs_rank_tracker_overview","ahrefs_rank_tracker_serp_overview","ahrefs_refdomains_history","ahrefs_referring_domains","ahrefs_related_terms","ahrefs_site_audit_page_explorer","ahrefs_top_pages","airtable_create_records","airtable_delete_records","airtable_get_base_schema","airtable_get_record","airtable_list_bases","airtable_list_records","airtable_list_tables","airtable_update_multiple_records","airtable_update_record","airtable_upsert_records","airweave_search","algolia_add_record","algolia_batch_operations","algolia_browse_records","algolia_clear_records","algolia_copy_move_index","algolia_delete_by_filter","algolia_delete_index","algolia_delete_record","algolia_get_record","algolia_get_records","algolia_get_settings","algolia_get_task_status","algolia_list_indices","algolia_partial_update_record","algolia_search","algolia_update_settings","amplitude_event_segmentation","amplitude_funnels","amplitude_get_active_users","amplitude_get_revenue","amplitude_group_identify","amplitude_identify_user","amplitude_list_events","amplitude_realtime_active_users","amplitude_retention","amplitude_send_event","amplitude_user_activity","amplitude_user_profile","amplitude_user_search","apify_get_dataset_items","apify_get_run","apify_run_actor_async","apify_run_actor_sync","apify_run_task","apollo_account_bulk_create","apollo_account_bulk_update","apollo_account_create","apollo_account_search","apollo_account_update","apollo_contact_bulk_create","apollo_contact_bulk_update","apollo_contact_create","apollo_contact_search","apollo_contact_update","apollo_email_accounts","apollo_opportunity_create","apollo_opportunity_get","apollo_opportunity_search","apollo_opportunity_update","apollo_organization_bulk_enrich","apollo_organization_enrich","apollo_organization_search","apollo_people_bulk_enrich","apollo_people_enrich","apollo_people_search","apollo_sequence_add_contacts","apollo_sequence_search","apollo_task_create","apollo_task_search","appconfig_create_application","appconfig_create_configuration_profile","appconfig_create_environment","appconfig_create_hosted_configuration_version","appconfig_delete_application","appconfig_delete_configuration_profile","appconfig_delete_environment","appconfig_delete_hosted_configuration_version","appconfig_get_application","appconfig_get_configuration","appconfig_get_configuration_profile","appconfig_get_deployment","appconfig_get_environment","appconfig_get_hosted_configuration_version","appconfig_list_applications","appconfig_list_configuration_profiles","appconfig_list_deployment_strategies","appconfig_list_deployments","appconfig_list_environments","appconfig_list_hosted_configuration_versions","appconfig_start_deployment","appconfig_stop_deployment","appconfig_update_application","appconfig_update_configuration_profile","appconfig_update_environment","arxiv_get_author_papers","arxiv_get_paper","arxiv_search","asana_add_comment","asana_add_followers","asana_create_project","asana_create_section","asana_create_subtask","asana_create_task","asana_delete_task","asana_get_project","asana_get_projects","asana_get_task","asana_list_sections","asana_list_workspaces","asana_search_tasks","asana_update_task","ashby_add_candidate_tag","ashby_anonymize_candidate","ashby_change_application_source","ashby_change_application_stage","ashby_create_application","ashby_create_candidate","ashby_create_note","ashby_delete_application","ashby_get_application","ashby_get_candidate","ashby_get_job","ashby_get_job_posting","ashby_get_offer","ashby_list_applications","ashby_list_archive_reasons","ashby_list_candidate_tags","ashby_list_candidates","ashby_list_custom_fields","ashby_list_departments","ashby_list_interviews","ashby_list_job_postings","ashby_list_jobs","ashby_list_locations","ashby_list_notes","ashby_list_offers","ashby_list_openings","ashby_list_sources","ashby_list_users","ashby_remove_candidate_tag","ashby_search_candidates","ashby_set_custom_field_value","ashby_set_custom_field_values","ashby_update_candidate","athena_batch_get_query_execution","athena_create_named_query","athena_delete_named_query","athena_get_named_query","athena_get_query_execution","athena_get_query_results","athena_list_databases","athena_list_named_queries","athena_list_query_executions","athena_list_table_metadata","athena_start_query","athena_stop_query","attio_assert_record","attio_create_attribute","attio_create_comment","attio_create_list","attio_create_list_entry","attio_create_note","attio_create_object","attio_create_record","attio_create_task","attio_create_webhook","attio_delete_comment","attio_delete_list_entry","attio_delete_note","attio_delete_record","attio_delete_task","attio_delete_webhook","attio_get_attribute","attio_get_comment","attio_get_list","attio_get_list_entry","attio_get_member","attio_get_note","attio_get_object","attio_get_record","attio_get_task","attio_get_thread","attio_get_webhook","attio_list_attributes","attio_list_lists","attio_list_members","attio_list_notes","attio_list_objects","attio_list_records","attio_list_tasks","attio_list_threads","attio_list_webhooks","attio_query_list_entries","attio_search_records","attio_update_attribute","attio_update_list","attio_update_list_entry","attio_update_object","attio_update_record","attio_update_task","attio_update_webhook","azure_data_explorer_create_table","azure_data_explorer_drop_table","azure_data_explorer_ingest_from_query","azure_data_explorer_ingest_inline","azure_data_explorer_list_databases","azure_data_explorer_list_functions","azure_data_explorer_list_tables","azure_data_explorer_management","azure_data_explorer_query","azure_data_explorer_show_database_schema","azure_data_explorer_show_ingestion_failures","azure_data_explorer_show_operations","azure_data_explorer_show_table_details","azure_data_explorer_show_table_schema","azure_devops_add_comment","azure_devops_create_work_item","azure_devops_get_build_log","azure_devops_get_build_timeline","azure_devops_get_comments","azure_devops_get_pipeline","azure_devops_get_pipeline_run","azure_devops_get_work_item","azure_devops_get_work_items_batch","azure_devops_get_work_items_between_builds","azure_devops_list_build_logs","azure_devops_list_builds","azure_devops_list_pipeline_runs","azure_devops_list_pipelines","azure_devops_query_work_items","azure_devops_update_work_item","box_copy_file","box_create_folder","box_delete_file","box_delete_folder","box_download_file","box_get_file_info","box_list_folder_items","box_search","box_sign_cancel_request","box_sign_create_request","box_sign_get_request","box_sign_list_requests","box_sign_resend_request","box_update_file","box_upload_file","brandfetch_get_brand","brandfetch_search","brex_archive_budget","brex_create_budget","brex_create_spend_limit","brex_create_transfer","brex_create_vendor","brex_get_budget","brex_get_cash_account","brex_get_company","brex_get_current_user","brex_get_expense","brex_get_spend_limit","brex_get_transfer","brex_get_user","brex_get_vendor","brex_list_budgets","brex_list_card_accounts","brex_list_card_statements","brex_list_card_transactions","brex_list_cards","brex_list_cash_accounts","brex_list_cash_statements","brex_list_cash_transactions","brex_list_departments","brex_list_expenses","brex_list_locations","brex_list_spend_limits","brex_list_titles","brex_list_transfers","brex_list_users","brex_list_vendors","brex_match_receipt","brex_update_expense","brex_update_vendor","brex_upload_receipt","brightdata_cancel_snapshot","brightdata_discover","brightdata_download_snapshot","brightdata_scrape_dataset","brightdata_scrape_url","brightdata_serp_search","brightdata_snapshot_status","brightdata_sync_scrape","browser_use_run_task","buffer_create_idea","buffer_create_post","buffer_delete_post","buffer_edit_post","buffer_get_account","buffer_get_channels","buffer_get_idea_groups","buffer_get_ideas","buffer_get_post","buffer_get_posts","calcom_cancel_booking","calcom_confirm_booking","calcom_create_booking","calcom_create_event_type","calcom_create_schedule","calcom_decline_booking","calcom_delete_event_type","calcom_delete_schedule","calcom_get_booking","calcom_get_default_schedule","calcom_get_event_type","calcom_get_schedule","calcom_get_slots","calcom_list_bookings","calcom_list_event_types","calcom_list_schedules","calcom_reschedule_booking","calcom_update_event_type","calcom_update_schedule","calendly_cancel_event","calendly_create_event_invitee","calendly_create_invitee_no_show","calendly_create_scheduling_link","calendly_create_webhook","calendly_delete_invitee_no_show","calendly_delete_webhook","calendly_get_current_user","calendly_get_event_invitee","calendly_get_event_type","calendly_get_scheduled_event","calendly_get_user","calendly_list_event_invitees","calendly_list_event_type_available_times","calendly_list_event_types","calendly_list_organization_memberships","calendly_list_routing_form_submissions","calendly_list_routing_forms","calendly_list_scheduled_events","calendly_list_user_availability_schedules","calendly_list_user_busy_times","calendly_list_webhooks","clay_populate","clerk_add_organization_member","clerk_ban_user","clerk_create_actor_token","clerk_create_allowlist_identifier","clerk_create_blocklist_identifier","clerk_create_organization","clerk_create_organization_invitation","clerk_create_user","clerk_delete_allowlist_identifier","clerk_delete_blocklist_identifier","clerk_delete_organization","clerk_delete_user","clerk_get_jwt_template","clerk_get_organization","clerk_get_session","clerk_get_user","clerk_get_user_oauth_token","clerk_list_allowlist_identifiers","clerk_list_blocklist_identifiers","clerk_list_jwt_templates","clerk_list_organization_invitations","clerk_list_organization_memberships","clerk_list_organizations","clerk_list_sessions","clerk_list_users","clerk_lock_user","clerk_remove_organization_member","clerk_revoke_actor_token","clerk_revoke_session","clerk_unban_user","clerk_unlock_user","clerk_update_organization","clerk_update_organization_membership","clerk_update_user","clickhouse_count_rows","clickhouse_create_database","clickhouse_create_table","clickhouse_delete","clickhouse_describe_table","clickhouse_drop_database","clickhouse_drop_partition","clickhouse_drop_table","clickhouse_execute","clickhouse_insert","clickhouse_insert_rows","clickhouse_introspect","clickhouse_kill_query","clickhouse_list_clusters","clickhouse_list_databases","clickhouse_list_mutations","clickhouse_list_partitions","clickhouse_list_running_queries","clickhouse_list_tables","clickhouse_optimize_table","clickhouse_query","clickhouse_rename_table","clickhouse_show_create_table","clickhouse_table_stats","clickhouse_truncate_table","clickhouse_update","clickup_add_tag_to_task","clickup_create_checklist","clickup_create_checklist_item","clickup_create_comment","clickup_create_folder","clickup_create_list","clickup_create_task","clickup_create_time_entry","clickup_delete_checklist","clickup_delete_checklist_item","clickup_delete_comment","clickup_delete_task","clickup_delete_time_entry","clickup_get_comments","clickup_get_custom_fields","clickup_get_folders","clickup_get_list_members","clickup_get_lists","clickup_get_running_timer","clickup_get_space_tags","clickup_get_spaces","clickup_get_task","clickup_get_task_members","clickup_get_tasks","clickup_get_time_entries","clickup_get_workspaces","clickup_remove_custom_field_value","clickup_remove_tag_from_task","clickup_search_tasks","clickup_set_custom_field_value","clickup_start_timer","clickup_stop_timer","clickup_update_checklist","clickup_update_checklist_item","clickup_update_comment","clickup_update_task","clickup_update_time_entry","clickup_upload_attachment","cloudflare_create_access_application","cloudflare_create_access_policy","cloudflare_create_access_service_token","cloudflare_create_dns_record","cloudflare_create_r2_bucket","cloudflare_create_rate_limit_rule","cloudflare_create_ruleset","cloudflare_create_ruleset_rule","cloudflare_create_zone","cloudflare_delete_access_application","cloudflare_delete_access_policy","cloudflare_delete_dns_record","cloudflare_delete_r2_bucket","cloudflare_delete_ruleset_rule","cloudflare_delete_zone","cloudflare_dns_analytics","cloudflare_get_access_application","cloudflare_get_r2_bucket","cloudflare_get_ruleset","cloudflare_get_ruleset_entrypoint","cloudflare_get_tunnel","cloudflare_get_tunnel_configuration","cloudflare_get_worker_script_settings","cloudflare_get_zone","cloudflare_get_zone_settings","cloudflare_list_access_applications","cloudflare_list_access_groups","cloudflare_list_access_identity_providers","cloudflare_list_access_policies","cloudflare_list_access_service_tokens","cloudflare_list_certificates","cloudflare_list_dns_records","cloudflare_list_managed_ruleset_overrides","cloudflare_list_r2_buckets","cloudflare_list_rate_limit_rules","cloudflare_list_rulesets","cloudflare_list_tunnels","cloudflare_list_worker_routes","cloudflare_list_worker_scripts","cloudflare_list_zones","cloudflare_purge_cache","cloudflare_revoke_access_service_token","cloudflare_update_access_application","cloudflare_update_access_policy","cloudflare_update_dns_record","cloudflare_update_rate_limit_rule","cloudflare_update_ruleset_rule","cloudflare_update_zone_setting","cloudformation_cancel_update_stack","cloudformation_create_change_set","cloudformation_create_stack","cloudformation_delete_stack","cloudformation_describe_change_set","cloudformation_describe_stack_drift_detection_status","cloudformation_describe_stack_events","cloudformation_describe_stacks","cloudformation_detect_stack_drift","cloudformation_execute_change_set","cloudformation_get_template","cloudformation_get_template_summary","cloudformation_list_stack_resources","cloudformation_update_stack","cloudformation_validate_template","cloudwatch_describe_alarm_history","cloudwatch_describe_alarms","cloudwatch_describe_log_groups","cloudwatch_describe_log_streams","cloudwatch_filter_log_events","cloudwatch_get_log_events","cloudwatch_get_metric_statistics","cloudwatch_list_metrics","cloudwatch_mute_alarm","cloudwatch_put_log_group_retention","cloudwatch_put_metric_data","cloudwatch_query_logs","cloudwatch_unmute_alarm","codepipeline_disable_stage_transition","codepipeline_enable_stage_transition","codepipeline_get_pipeline","codepipeline_get_pipeline_execution","codepipeline_get_pipeline_state","codepipeline_list_action_executions","codepipeline_list_pipeline_executions","codepipeline_list_pipelines","codepipeline_put_approval_result","codepipeline_retry_stage_execution","codepipeline_start_execution","codepipeline_stop_execution","confluence_add_label","confluence_create_blogpost","confluence_create_comment","confluence_create_page","confluence_create_page_property","confluence_create_space","confluence_create_space_property","confluence_delete_attachment","confluence_delete_blogpost","confluence_delete_comment","confluence_delete_label","confluence_delete_page","confluence_delete_page_property","confluence_delete_space","confluence_delete_space_property","confluence_get_blogpost","confluence_get_page_ancestors","confluence_get_page_children","confluence_get_page_descendants","confluence_get_page_version","confluence_get_pages_by_label","confluence_get_space","confluence_get_task","confluence_get_user","confluence_list_attachments","confluence_list_blogposts","confluence_list_blogposts_in_space","confluence_list_comments","confluence_list_labels","confluence_list_page_properties","confluence_list_page_versions","confluence_list_pages_in_space","confluence_list_space_labels","confluence_list_space_permissions","confluence_list_space_properties","confluence_list_spaces","confluence_list_tasks","confluence_retrieve","confluence_search","confluence_search_in_space","confluence_update","confluence_update_blogpost","confluence_update_comment","confluence_update_space","confluence_update_task","confluence_upload_attachment","context_dev_classify_naics","context_dev_classify_sic","context_dev_crawl","context_dev_extract","context_dev_extract_product","context_dev_extract_products","context_dev_get_brand","context_dev_get_brand_by_email","context_dev_get_brand_by_name","context_dev_get_brand_by_ticker","context_dev_identify_transaction","context_dev_map","context_dev_scrape_fonts","context_dev_scrape_html","context_dev_scrape_images","context_dev_scrape_markdown","context_dev_scrape_styleguide","context_dev_screenshot","context_dev_search","convex_action","convex_document_deltas","convex_list_documents","convex_list_tables","convex_mutation","convex_query","convex_run_function","crowdstrike_create_indicators","crowdstrike_delete_indicators","crowdstrike_delete_rtr_session","crowdstrike_execute_rtr_command","crowdstrike_get_alert_details","crowdstrike_get_case_details","crowdstrike_get_host_group_details","crowdstrike_get_indicator_details","crowdstrike_get_rtr_command_status","crowdstrike_get_sensor_aggregates","crowdstrike_get_sensor_details","crowdstrike_get_vulnerability_details","crowdstrike_init_rtr_session","crowdstrike_perform_host_action","crowdstrike_perform_host_group_action","crowdstrike_query_alerts","crowdstrike_query_cases","crowdstrike_query_host_groups","crowdstrike_query_indicators","crowdstrike_query_sensors","crowdstrike_query_vulnerabilities","crowdstrike_update_alerts","crowdstrike_update_indicators","cursor_add_followup","cursor_add_followup_v2","cursor_delete_agent","cursor_delete_agent_v2","cursor_download_artifact","cursor_download_artifact_v2","cursor_get_agent","cursor_get_agent_v2","cursor_get_api_key_info","cursor_get_api_key_info_v2","cursor_get_conversation","cursor_get_conversation_v2","cursor_launch_agent","cursor_launch_agent_v2","cursor_list_agents","cursor_list_agents_v2","cursor_list_artifacts","cursor_list_artifacts_v2","cursor_list_models","cursor_list_models_v2","cursor_list_repositories","cursor_list_repositories_v2","cursor_stop_agent","cursor_stop_agent_v2","dagster_delete_run","dagster_get_asset","dagster_get_run","dagster_get_run_logs","dagster_launch_run","dagster_list_assets","dagster_list_jobs","dagster_list_runs","dagster_list_schedules","dagster_list_sensors","dagster_materialize_assets","dagster_reexecute_run","dagster_report_asset_materialization","dagster_start_schedule","dagster_start_sensor","dagster_stop_schedule","dagster_stop_sensor","dagster_terminate_run","dagster_wipe_asset","databricks_cancel_run","databricks_execute_sql","databricks_get_cluster","databricks_get_job","databricks_get_run","databricks_get_run_output","databricks_get_statement","databricks_list_clusters","databricks_list_jobs","databricks_list_runs","databricks_list_warehouses","databricks_run_job","datadog_add_incident_todo","datadog_cancel_downtime","datadog_create_dashboard","datadog_create_downtime","datadog_create_event","datadog_create_incident","datadog_create_monitor","datadog_create_slo","datadog_delete_dashboard","datadog_delete_slo","datadog_get_browser_synthetics_results","datadog_get_dashboard","datadog_get_incident","datadog_get_monitor","datadog_get_security_signal","datadog_get_slo","datadog_get_slo_history","datadog_get_synthetics_results","datadog_get_synthetics_test","datadog_list_dashboards","datadog_list_downtimes","datadog_list_incidents","datadog_list_monitors","datadog_list_security_rules","datadog_list_security_signals","datadog_list_services","datadog_list_slos","datadog_list_synthetics_tests","datadog_mute_monitor","datadog_query_logs","datadog_query_timeseries","datadog_search_spans","datadog_send_logs","datadog_submit_metrics","datadog_trigger_synthetics_tests","datadog_unmute_monitor","datadog_update_incident","datadog_update_security_signal_assignee","datadog_update_security_signal_state","datadog_update_slo","datadog_update_synthetics_status","datagma_enrich_company","datagma_enrich_person","datagma_find_email","datagma_find_phone","datagma_get_credits","daytona_create_sandbox","daytona_delete_sandbox","daytona_download_file","daytona_execute_command","daytona_get_sandbox","daytona_git_clone","daytona_list_files","daytona_list_sandboxes","daytona_run_code","daytona_start_sandbox","daytona_stop_sandbox","daytona_upload_file","deployed_block_executor","deployments_deploy","deployments_get_version","deployments_list_versions","deployments_promote","deployments_undeploy","devin_append_session_tags","devin_archive_session","devin_create_session","devin_get_session","devin_get_session_tags","devin_list_session_attachments","devin_list_session_messages","devin_list_sessions","devin_replace_session_tags","devin_send_message","devin_terminate_session","discord_add_reaction","discord_archive_thread","discord_assign_role","discord_ban_member","discord_bulk_delete_messages","discord_create_channel","discord_create_invite","discord_create_role","discord_create_thread","discord_create_webhook","discord_delete_channel","discord_delete_invite","discord_delete_message","discord_delete_role","discord_delete_webhook","discord_edit_message","discord_execute_webhook","discord_get_channel","discord_get_invite","discord_get_member","discord_get_messages","discord_get_pinned_messages","discord_get_server","discord_get_user","discord_get_webhook","discord_join_thread","discord_kick_member","discord_leave_thread","discord_list_channels","discord_list_roles","discord_pin_message","discord_remove_reaction","discord_remove_role","discord_send_message","discord_unban_member","discord_unpin_message","discord_update_channel","discord_update_member","discord_update_role","docusign_create_from_template","docusign_download_document","docusign_get_envelope","docusign_list_envelopes","docusign_list_recipients","docusign_list_templates","docusign_send_envelope","docusign_void_envelope","downdetector_get_company","downdetector_get_company_attribution","downdetector_get_company_baseline","downdetector_get_company_events","downdetector_get_company_incidents","downdetector_get_company_indicators","downdetector_get_company_last_15","downdetector_get_company_status","downdetector_get_provider","downdetector_get_reports","downdetector_get_site_companies","downdetector_list_categories","downdetector_list_incidents","downdetector_list_sites","downdetector_search_companies","dropbox_copy","dropbox_create_folder","dropbox_create_shared_link","dropbox_delete","dropbox_download","dropbox_get_metadata","dropbox_list_folder","dropbox_list_revisions","dropbox_list_shared_links","dropbox_move","dropbox_restore","dropbox_search","dropbox_upload","dropcontact_enrich_contact","dspy_chain_of_thought","dspy_predict","dspy_react","dub_bulk_create_links","dub_bulk_delete_links","dub_bulk_update_links","dub_create_link","dub_create_tag","dub_delete_link","dub_get_analytics","dub_get_events","dub_get_link","dub_get_links_count","dub_get_qr_code","dub_list_domains","dub_list_folders","dub_list_links","dub_list_tags","dub_update_link","dub_upsert_link","duckduckgo_search","dynamodb_delete","dynamodb_get","dynamodb_introspect","dynamodb_put","dynamodb_query","dynamodb_scan","dynamodb_update","dynatrace_add_problem_comment","dynatrace_add_tags","dynatrace_close_problem","dynatrace_create_settings_object","dynatrace_create_slo","dynatrace_delete_problem_comment","dynatrace_delete_settings_object","dynatrace_delete_slo","dynatrace_delete_tag","dynatrace_execute_synthetic_monitors","dynatrace_get_attack","dynatrace_get_audit_logs","dynatrace_get_entity","dynatrace_get_event","dynatrace_get_metric","dynatrace_get_problem","dynatrace_get_problem_comment","dynatrace_get_security_problem","dynatrace_get_settings_object","dynatrace_get_slo","dynatrace_get_synthetic_batch","dynatrace_ingest_event","dynatrace_ingest_logs","dynatrace_ingest_metrics","dynatrace_list_attacks","dynatrace_list_entities","dynatrace_list_entity_types","dynatrace_list_events","dynatrace_list_metrics","dynatrace_list_problem_comments","dynatrace_list_problems","dynatrace_list_remediation_items","dynatrace_list_security_problems","dynatrace_list_settings_objects","dynatrace_list_settings_schemas","dynatrace_list_slos","dynatrace_list_synthetic_monitors","dynatrace_list_tags","dynatrace_mute_security_problem","dynatrace_mute_security_problems","dynatrace_query_metrics","dynatrace_search_logs","dynatrace_unmute_security_problem","dynatrace_unmute_security_problems","dynatrace_update_problem_comment","dynatrace_update_settings_object","dynatrace_update_slo","elasticsearch_bulk","elasticsearch_cluster_health","elasticsearch_cluster_stats","elasticsearch_count","elasticsearch_create_index","elasticsearch_delete_document","elasticsearch_delete_index","elasticsearch_get_document","elasticsearch_get_index","elasticsearch_index_document","elasticsearch_list_indices","elasticsearch_search","elasticsearch_update_document","elevenlabs_audio_isolation","elevenlabs_edit_voice_settings","elevenlabs_get_user","elevenlabs_get_voice","elevenlabs_get_voice_settings","elevenlabs_list_models","elevenlabs_list_voices","elevenlabs_sound_effects","elevenlabs_speech_to_speech","elevenlabs_tts","emailbison_attach_leads_to_campaign","emailbison_attach_tags_to_leads","emailbison_create_campaign","emailbison_create_lead","emailbison_create_tag","emailbison_get_lead","emailbison_list_campaigns","emailbison_list_leads","emailbison_list_replies","emailbison_list_tags","emailbison_update_campaign","emailbison_update_campaign_status","emailbison_update_lead","embeddings_cohere","embeddings_gemini","embeddings_mistral","embeddings_openai","embeddings_openrouter","enrich_check_credits","enrich_company_funding","enrich_company_lookup","enrich_company_revenue","enrich_disposable_email_check","enrich_email_to_ip","enrich_email_to_person_lite","enrich_email_to_phone","enrich_email_to_profile","enrich_find_email","enrich_get_post_details","enrich_ip_to_company","enrich_linkedin_profile","enrich_linkedin_to_personal_email","enrich_linkedin_to_work_email","enrich_phone_finder","enrich_reverse_hash_lookup","enrich_sales_pointer_people","enrich_search_company","enrich_search_company_activities","enrich_search_company_employees","enrich_search_jobs","enrich_search_logo","enrich_search_people","enrich_search_people_activities","enrich_search_post_comments","enrich_search_post_comments_by_url","enrich_search_post_reactions","enrich_search_post_reactions_by_url","enrich_search_posts","enrich_search_similar_companies","enrich_verify_email","enrichment_run","enrow_find_email","enrow_verify_email","evernote_copy_note","evernote_create_note","evernote_create_notebook","evernote_create_tag","evernote_delete_note","evernote_get_note","evernote_get_notebook","evernote_list_notebooks","evernote_list_tags","evernote_search_notes","evernote_update_note","exa_agent","exa_answer","exa_find_similar_links","exa_get_contents","exa_search","extend_parser","extend_parser_v2","fathom_get_summary","fathom_get_transcript","fathom_list_meeting_types","fathom_list_meetings","fathom_list_team_members","fathom_list_teams","file_append","file_compress","file_decompress","file_fetch","file_get","file_get_content","file_manage_sharing","file_parser","file_parser_v2","file_parser_v3","file_read","file_write","findymail_find_email_from_linkedin","findymail_find_email_from_name","findymail_find_emails_by_domain","findymail_find_employees","findymail_find_phone","findymail_get_company","findymail_get_credits","findymail_lookup_technologies","findymail_reverse_email_lookup","findymail_search_technologies","findymail_verify_email","firecrawl_agent","firecrawl_batch_scrape","firecrawl_batch_scrape_status","firecrawl_cancel_crawl","firecrawl_crawl","firecrawl_crawl_status","firecrawl_credit_usage","firecrawl_extract","firecrawl_extract_status","firecrawl_map","firecrawl_parse","firecrawl_scrape","firecrawl_search","fireflies_add_to_live_meeting","fireflies_create_bite","fireflies_delete_transcript","fireflies_get_transcript","fireflies_get_user","fireflies_list_bites","fireflies_list_contacts","fireflies_list_transcripts","fireflies_list_users","fireflies_upload_audio","flint_create_task","flint_generate_pages","flint_get_task","function_execute","gamma_check_status","gamma_generate","gamma_generate_from_template","gamma_list_folders","gamma_list_themes","github_add_assignees","github_add_assignees_v2","github_add_labels","github_add_labels_v2","github_cancel_workflow_run","github_cancel_workflow_run_v2","github_check_star","github_check_star_v2","github_close_issue","github_close_issue_v2","github_close_pr","github_close_pr_v2","github_comment","github_comment_v2","github_compare_commits","github_compare_commits_v2","github_create_branch","github_create_branch_v2","github_create_comment_reaction","github_create_comment_reaction_v2","github_create_file","github_create_file_v2","github_create_gist","github_create_gist_v2","github_create_issue","github_create_issue_reaction","github_create_issue_reaction_v2","github_create_issue_v2","github_create_milestone","github_create_milestone_v2","github_create_pr","github_create_pr_review","github_create_pr_review_v2","github_create_pr_v2","github_create_project","github_create_project_v2","github_create_release","github_create_release_v2","github_delete_branch","github_delete_branch_v2","github_delete_comment","github_delete_comment_reaction","github_delete_comment_reaction_v2","github_delete_comment_v2","github_delete_file","github_delete_file_v2","github_delete_gist","github_delete_gist_v2","github_delete_issue_reaction","github_delete_issue_reaction_v2","github_delete_milestone","github_delete_milestone_v2","github_delete_project","github_delete_project_v2","github_delete_release","github_delete_release_v2","github_fork_gist","github_fork_gist_v2","github_fork_repo","github_fork_repo_v2","github_get_branch","github_get_branch_protection","github_get_branch_protection_v2","github_get_branch_v2","github_get_commit","github_get_commit_v2","github_get_file_content","github_get_file_content_v2","github_get_gist","github_get_gist_v2","github_get_issue","github_get_issue_v2","github_get_latest_release","github_get_latest_release_v2","github_get_milestone","github_get_milestone_v2","github_get_pr_files","github_get_pr_files_v2","github_get_project","github_get_project_v2","github_get_readme","github_get_readme_v2","github_get_release","github_get_release_v2","github_get_tree","github_get_tree_v2","github_get_workflow","github_get_workflow_run","github_get_workflow_run_v2","github_get_workflow_v2","github_issue_comment","github_issue_comment_v2","github_job_logs","github_latest_commit","github_latest_commit_v2","github_list_branches","github_list_branches_v2","github_list_commits","github_list_commits_v2","github_list_forks","github_list_forks_v2","github_list_gists","github_list_gists_v2","github_list_issue_comments","github_list_issue_comments_v2","github_list_issues","github_list_issues_v2","github_list_milestones","github_list_milestones_v2","github_list_pr_comments","github_list_pr_comments_v2","github_list_projects","github_list_projects_v2","github_list_prs","github_list_prs_v2","github_list_releases","github_list_releases_v2","github_list_review_threads","github_list_stargazers","github_list_stargazers_v2","github_list_tags","github_list_tags_v2","github_list_workflow_runs","github_list_workflow_runs_v2","github_list_workflows","github_list_workflows_v2","github_merge_pr","github_merge_pr_v2","github_pr","github_pr_v2","github_remove_label","github_remove_label_v2","github_reply_review_thread","github_repo_info","github_repo_info_v2","github_request_reviewers","github_request_reviewers_v2","github_rerun_workflow","github_rerun_workflow_v2","github_resolve_review_thread","github_search_code","github_search_code_v2","github_search_commits","github_search_commits_v2","github_search_issues","github_search_issues_v2","github_search_repos","github_search_repos_v2","github_search_users","github_search_users_v2","github_star_gist","github_star_gist_v2","github_star_repo","github_star_repo_v2","github_status_check_rollup","github_trigger_workflow","github_trigger_workflow_v2","github_unstar_gist","github_unstar_gist_v2","github_unstar_repo","github_unstar_repo_v2","github_update_branch_protection","github_update_branch_protection_v2","github_update_comment","github_update_comment_v2","github_update_file","github_update_file_v2","github_update_gist","github_update_gist_v2","github_update_issue","github_update_issue_v2","github_update_milestone","github_update_milestone_v2","github_update_pr","github_update_pr_v2","github_update_project","github_update_project_v2","github_update_release","github_update_release_v2","gitlab_activate_user","gitlab_add_member","gitlab_add_saml_group_link","gitlab_approve_access_request","gitlab_approve_merge_request","gitlab_approve_user","gitlab_ban_user","gitlab_block_user","gitlab_cancel_pipeline","gitlab_compare_branches","gitlab_create_branch","gitlab_create_file","gitlab_create_issue","gitlab_create_issue_note","gitlab_create_merge_request","gitlab_create_merge_request_note","gitlab_create_pipeline","gitlab_create_release","gitlab_create_user","gitlab_deactivate_user","gitlab_delete_branch","gitlab_delete_issue","gitlab_delete_saml_group_link","gitlab_delete_user","gitlab_delete_user_identity","gitlab_deny_access_request","gitlab_get_file","gitlab_get_group","gitlab_get_issue","gitlab_get_job_log","gitlab_get_merge_request","gitlab_get_merge_request_changes","gitlab_get_pipeline","gitlab_get_project","gitlab_invite_member","gitlab_list_access_requests","gitlab_list_branches","gitlab_list_commits","gitlab_list_groups","gitlab_list_invitations","gitlab_list_issues","gitlab_list_members","gitlab_list_merge_requests","gitlab_list_pipeline_jobs","gitlab_list_pipelines","gitlab_list_projects","gitlab_list_releases","gitlab_list_repository_tree","gitlab_list_saml_group_links","gitlab_list_user_memberships","gitlab_merge_merge_request","gitlab_play_job","gitlab_reject_user","gitlab_remove_member","gitlab_retry_pipeline","gitlab_revoke_invitation","gitlab_search_users","gitlab_unban_user","gitlab_unblock_user","gitlab_update_file","gitlab_update_invitation","gitlab_update_issue","gitlab_update_member","gitlab_update_merge_request","gitlab_update_user","gmail_add_label","gmail_add_label_v2","gmail_archive","gmail_archive_v2","gmail_create_label_v2","gmail_delete","gmail_delete_draft_v2","gmail_delete_label_v2","gmail_delete_v2","gmail_draft","gmail_draft_v2","gmail_edit_draft_v2","gmail_get_draft_v2","gmail_get_thread_v2","gmail_list_drafts_v2","gmail_list_labels_v2","gmail_list_threads_v2","gmail_mark_read","gmail_mark_read_v2","gmail_mark_unread","gmail_mark_unread_v2","gmail_move","gmail_move_v2","gmail_read","gmail_read_v2","gmail_remove_label","gmail_remove_label_v2","gmail_search","gmail_search_v2","gmail_send","gmail_send_v2","gmail_trash_thread_v2","gmail_unarchive","gmail_unarchive_v2","gmail_untrash_thread_v2","gmail_update_label_v2","gong_aggregate_activity","gong_aggregate_by_period","gong_answered_scorecards","gong_ask_anything","gong_assign_flow_prospects","gong_create_call","gong_day_by_day_activity","gong_get_brief","gong_get_call","gong_get_call_transcript","gong_get_coaching","gong_get_extensive_calls","gong_get_folder_content","gong_get_logs","gong_get_prospect_flows","gong_get_user","gong_interaction_stats","gong_list_calls","gong_list_flows","gong_list_library_folders","gong_list_scorecards","gong_list_trackers","gong_list_users","gong_list_workspaces","gong_lookup_email","gong_lookup_phone","gong_purge_email_address","gong_purge_phone_number","gong_unassign_flow_prospects","google_ads_ad_performance","google_ads_campaign_performance","google_ads_list_ad_groups","google_ads_list_campaigns","google_ads_list_customers","google_ads_search","google_appsheet_add_rows","google_appsheet_delete_rows","google_appsheet_edit_rows","google_appsheet_find_rows","google_bigquery_create_dataset","google_bigquery_create_table","google_bigquery_delete_dataset","google_bigquery_delete_table","google_bigquery_get_query_results","google_bigquery_get_table","google_bigquery_insert_rows","google_bigquery_list_datasets","google_bigquery_list_table_data","google_bigquery_list_tables","google_bigquery_query","google_books_volume_details","google_books_volume_search","google_calendar_create","google_calendar_create_calendar","google_calendar_create_calendar_v2","google_calendar_create_v2","google_calendar_delete","google_calendar_delete_calendar","google_calendar_delete_calendar_v2","google_calendar_delete_v2","google_calendar_freebusy","google_calendar_freebusy_v2","google_calendar_get","google_calendar_get_v2","google_calendar_instances","google_calendar_instances_v2","google_calendar_invite","google_calendar_invite_v2","google_calendar_list","google_calendar_list_acl","google_calendar_list_acl_v2","google_calendar_list_calendars","google_calendar_list_calendars_v2","google_calendar_list_v2","google_calendar_move","google_calendar_move_v2","google_calendar_quick_add","google_calendar_quick_add_v2","google_calendar_share_calendar","google_calendar_share_calendar_v2","google_calendar_unshare_calendar","google_calendar_unshare_calendar_v2","google_calendar_update","google_calendar_update_acl","google_calendar_update_acl_v2","google_calendar_update_calendar","google_calendar_update_calendar_v2","google_calendar_update_v2","google_contacts_create","google_contacts_delete","google_contacts_get","google_contacts_list","google_contacts_search","google_contacts_update","google_docs_create","google_docs_create_named_range","google_docs_create_paragraph_bullets","google_docs_delete_content_range","google_docs_delete_named_range","google_docs_delete_paragraph_bullets","google_docs_insert_image","google_docs_insert_page_break","google_docs_insert_table","google_docs_insert_text","google_docs_read","google_docs_replace_text","google_docs_update_paragraph_style","google_docs_update_text_style","google_docs_write","google_drive_copy","google_drive_create_comment","google_drive_create_folder","google_drive_delete","google_drive_delete_comment","google_drive_download","google_drive_export","google_drive_get_about","google_drive_get_content","google_drive_get_file","google_drive_get_revision","google_drive_list","google_drive_list_comments","google_drive_list_permissions","google_drive_list_revisions","google_drive_move","google_drive_search","google_drive_share","google_drive_trash","google_drive_unshare","google_drive_untrash","google_drive_update","google_drive_upload","google_forms_batch_update","google_forms_create_form","google_forms_create_watch","google_forms_delete_watch","google_forms_get_form","google_forms_get_responses","google_forms_list_watches","google_forms_renew_watch","google_forms_set_publish_settings","google_groups_add_alias","google_groups_add_member","google_groups_create_group","google_groups_delete_group","google_groups_get_group","google_groups_get_member","google_groups_get_settings","google_groups_has_member","google_groups_list_aliases","google_groups_list_groups","google_groups_list_members","google_groups_remove_alias","google_groups_remove_member","google_groups_update_group","google_groups_update_member","google_groups_update_settings","google_maps_air_quality","google_maps_directions","google_maps_distance_matrix","google_maps_elevation","google_maps_geocode","google_maps_geolocate","google_maps_place_details","google_maps_places_nearby","google_maps_places_search","google_maps_pollen","google_maps_reverse_geocode","google_maps_snap_to_roads","google_maps_solar","google_maps_speed_limits","google_maps_timezone","google_maps_validate_address","google_meet_create_space","google_meet_end_conference","google_meet_get_conference_record","google_meet_get_space","google_meet_list_conference_records","google_meet_list_participants","google_pagespeed_analyze","google_search","google_sheets_append","google_sheets_append_v2","google_sheets_batch_clear_v2","google_sheets_batch_get_v2","google_sheets_batch_update_v2","google_sheets_clear_v2","google_sheets_copy_sheet_v2","google_sheets_create_spreadsheet_v2","google_sheets_delete_rows_v2","google_sheets_delete_sheet_v2","google_sheets_delete_spreadsheet_v2","google_sheets_get_spreadsheet_v2","google_sheets_read","google_sheets_read_v2","google_sheets_update","google_sheets_update_v2","google_sheets_write","google_sheets_write_v2","google_slides_add_image","google_slides_add_slide","google_slides_batch_update","google_slides_copy_presentation","google_slides_create","google_slides_create_line","google_slides_create_paragraph_bullets","google_slides_create_shape","google_slides_create_sheets_chart","google_slides_create_table","google_slides_create_video","google_slides_delete_object","google_slides_delete_paragraph_bullets","google_slides_delete_table_column","google_slides_delete_table_row","google_slides_delete_text","google_slides_duplicate_object","google_slides_export_presentation","google_slides_get_page","google_slides_get_thumbnail","google_slides_group_objects","google_slides_insert_table_columns","google_slides_insert_table_rows","google_slides_insert_text","google_slides_merge_table_cells","google_slides_read","google_slides_refresh_sheets_chart","google_slides_replace_all_shapes_with_image","google_slides_replace_all_shapes_with_sheets_chart","google_slides_replace_all_text","google_slides_replace_image","google_slides_reroute_line","google_slides_ungroup_objects","google_slides_unmerge_table_cells","google_slides_update_image_properties","google_slides_update_line_category","google_slides_update_line_properties","google_slides_update_page_element_alt_text","google_slides_update_page_element_transform","google_slides_update_page_elements_z_order","google_slides_update_page_properties","google_slides_update_paragraph_style","google_slides_update_shape_properties","google_slides_update_slide_properties","google_slides_update_slides_position","google_slides_update_table_border_properties","google_slides_update_table_cell_properties","google_slides_update_table_column_properties","google_slides_update_table_row_properties","google_slides_update_text_style","google_slides_update_video_properties","google_slides_write","google_tasks_create","google_tasks_delete","google_tasks_get","google_tasks_list","google_tasks_list_task_lists","google_tasks_update","google_translate_detect","google_translate_text","google_vault_add_held_accounts","google_vault_add_matters_permissions","google_vault_close_matters","google_vault_create_matters","google_vault_create_matters_export","google_vault_create_matters_holds","google_vault_create_saved_query","google_vault_delete_matters","google_vault_delete_matters_export","google_vault_delete_matters_holds","google_vault_delete_saved_query","google_vault_download_export_file","google_vault_list_matters","google_vault_list_matters_export","google_vault_list_matters_holds","google_vault_list_saved_queries","google_vault_remove_held_accounts","google_vault_remove_matters_permissions","google_vault_reopen_matters","google_vault_undelete_matters","google_vault_update_matters","google_vault_update_matters_holds","grafana_check_data_source_health","grafana_create_alert_rule","grafana_create_annotation","grafana_create_contact_point","grafana_create_dashboard","grafana_create_folder","grafana_delete_alert_rule","grafana_delete_annotation","grafana_delete_contact_point","grafana_delete_dashboard","grafana_delete_folder","grafana_get_alert_rule","grafana_get_alert_rule_group","grafana_get_dashboard","grafana_get_data_source","grafana_get_folder","grafana_get_health","grafana_list_alert_rules","grafana_list_annotations","grafana_list_contact_points","grafana_list_dashboards","grafana_list_data_sources","grafana_list_folders","grafana_move_folder","grafana_query_data_source","grafana_update_alert_rule","grafana_update_annotation","grafana_update_contact_point","grafana_update_dashboard","grafana_update_folder","grain_create_hook","grain_create_hook_v2","grain_delete_hook","grain_delete_hook_v2","grain_get_recording","grain_get_transcript","grain_list_hooks","grain_list_hooks_v2","grain_list_meeting_types","grain_list_recordings","grain_list_teams","grain_list_views","granola_get_note","granola_list_folders","granola_list_notes","greenhouse_get_application","greenhouse_get_candidate","greenhouse_get_job","greenhouse_get_user","greenhouse_list_applications","greenhouse_list_candidates","greenhouse_list_departments","greenhouse_list_job_stages","greenhouse_list_jobs","greenhouse_list_offices","greenhouse_list_users","greptile_index_repo","greptile_query","greptile_search","greptile_status","guardrails_validate","hex_cancel_run","hex_create_collection","hex_create_group","hex_deactivate_user","hex_delete_group","hex_get_collection","hex_get_data_connection","hex_get_group","hex_get_project","hex_get_project_runs","hex_get_queried_tables","hex_get_run_status","hex_list_collections","hex_list_data_connections","hex_list_groups","hex_list_projects","hex_list_users","hex_run_project","hex_update_collection","hex_update_group","hex_update_project","http_request","hubspot_add_list_memberships","hubspot_create_appointment","hubspot_create_association","hubspot_create_company","hubspot_create_contact","hubspot_create_deal","hubspot_create_email","hubspot_create_line_item","hubspot_create_list","hubspot_create_note","hubspot_create_ticket","hubspot_delete_association","hubspot_delete_company","hubspot_delete_contact","hubspot_delete_deal","hubspot_delete_line_item","hubspot_delete_ticket","hubspot_get_appointment","hubspot_get_association_labels","hubspot_get_cart","hubspot_get_company","hubspot_get_contact","hubspot_get_deal","hubspot_get_email","hubspot_get_line_item","hubspot_get_list","hubspot_get_list_memberships","hubspot_get_marketing_event","hubspot_get_note","hubspot_get_properties","hubspot_get_quote","hubspot_get_ticket","hubspot_get_users","hubspot_list_appointments","hubspot_list_associations","hubspot_list_carts","hubspot_list_companies","hubspot_list_contacts","hubspot_list_deals","hubspot_list_emails","hubspot_list_line_items","hubspot_list_lists","hubspot_list_marketing_events","hubspot_list_notes","hubspot_list_owners","hubspot_list_quotes","hubspot_list_tickets","hubspot_remove_list_memberships","hubspot_search_companies","hubspot_search_contacts","hubspot_search_deals","hubspot_search_emails","hubspot_search_line_items","hubspot_search_notes","hubspot_search_quotes","hubspot_search_tickets","hubspot_update_appointment","hubspot_update_company","hubspot_update_contact","hubspot_update_deal","hubspot_update_line_item","hubspot_update_ticket","huggingface_chat","hunter_companies_find","hunter_discover","hunter_domain_search","hunter_email_count","hunter_email_finder","hunter_email_verifier","iam_add_user_to_group","iam_attach_role_policy","iam_attach_user_policy","iam_create_access_key","iam_create_role","iam_create_user","iam_delete_access_key","iam_delete_role","iam_delete_user","iam_detach_role_policy","iam_detach_user_policy","iam_get_role","iam_get_user","iam_list_attached_role_policies","iam_list_attached_user_policies","iam_list_groups","iam_list_policies","iam_list_roles","iam_list_users","iam_remove_user_from_group","iam_simulate_principal_policy","icypeas_find_email","icypeas_verify_email","identity_center_check_assignment_deletion_status","identity_center_check_assignment_status","identity_center_create_account_assignment","identity_center_delete_account_assignment","identity_center_describe_account","identity_center_get_group","identity_center_get_user","identity_center_list_account_assignments","identity_center_list_accounts","identity_center_list_groups","identity_center_list_instances","identity_center_list_permission_sets","image_generate","incidentio_actions_create","incidentio_actions_list","incidentio_actions_show","incidentio_actions_update","incidentio_alert_events_create","incidentio_alerts_list","incidentio_alerts_resolve","incidentio_alerts_show","incidentio_catalog_entries_list","incidentio_catalog_types_list","incidentio_custom_fields_create","incidentio_custom_fields_delete","incidentio_custom_fields_list","incidentio_custom_fields_show","incidentio_custom_fields_update","incidentio_escalation_paths_create","incidentio_escalation_paths_delete","incidentio_escalation_paths_list","incidentio_escalation_paths_show","incidentio_escalation_paths_update","incidentio_escalations_cancel","incidentio_escalations_create","incidentio_escalations_list","incidentio_escalations_show","incidentio_follow_ups_create","incidentio_follow_ups_list","incidentio_follow_ups_show","incidentio_follow_ups_update","incidentio_incident_alerts_list","incidentio_incident_memberships_create","incidentio_incident_memberships_revoke","incidentio_incident_participants_list","incidentio_incident_roles_create","incidentio_incident_roles_delete","incidentio_incident_roles_list","incidentio_incident_roles_show","incidentio_incident_roles_update","incidentio_incident_statuses_list","incidentio_incident_timestamps_list","incidentio_incident_timestamps_show","incidentio_incident_types_list","incidentio_incident_updates_list","incidentio_incidents_create","incidentio_incidents_list","incidentio_incidents_show","incidentio_incidents_update","incidentio_on_call_now","incidentio_schedule_entries_list","incidentio_schedule_overrides_create","incidentio_schedule_overrides_list","incidentio_schedules_create","incidentio_schedules_delete","incidentio_schedules_list","incidentio_schedules_show","incidentio_schedules_update","incidentio_severities_list","incidentio_teams_list","incidentio_teams_show","incidentio_users_list","incidentio_users_show","incidentio_workflows_create","incidentio_workflows_delete","incidentio_workflows_list","incidentio_workflows_show","incidentio_workflows_update","infisical_create_secret","infisical_delete_secret","infisical_get_secret","infisical_list_secrets","infisical_update_secret","instagram_delete_comment","instagram_download_media","instagram_get_account_insights","instagram_get_container_status","instagram_get_conversation_messages","instagram_get_media","instagram_get_media_insights","instagram_get_message","instagram_get_profile","instagram_get_publishing_limit","instagram_hide_comment","instagram_list_comments","instagram_list_conversations","instagram_list_media","instagram_list_stories","instagram_private_reply","instagram_publish_carousel","instagram_publish_image","instagram_publish_reel","instagram_publish_story","instagram_publish_video","instagram_reply_to_comment","instagram_send_text_message","instagram_set_comments_enabled","instantly_activate_campaign","instantly_create_campaign","instantly_create_lead","instantly_create_lead_list","instantly_delete_campaign","instantly_delete_leads","instantly_get_lead","instantly_list_campaigns","instantly_list_emails","instantly_list_lead_lists","instantly_list_leads","instantly_patch_campaign","instantly_patch_lead","instantly_pause_campaign","instantly_reply_to_email","instantly_update_lead_interest_status","intercom_assign_conversation_v2","intercom_attach_contact_to_company_v2","intercom_close_conversation_v2","intercom_create_company","intercom_create_company_v2","intercom_create_contact","intercom_create_contact_v2","intercom_create_event_v2","intercom_create_message","intercom_create_message_v2","intercom_create_note_v2","intercom_create_tag_v2","intercom_create_ticket","intercom_create_ticket_v2","intercom_delete_contact","intercom_delete_contact_v2","intercom_detach_contact_from_company_v2","intercom_get_company","intercom_get_company_v2","intercom_get_contact","intercom_get_contact_v2","intercom_get_conversation","intercom_get_conversation_v2","intercom_get_ticket","intercom_get_ticket_v2","intercom_list_admins_v2","intercom_list_companies","intercom_list_companies_v2","intercom_list_contacts","intercom_list_contacts_v2","intercom_list_conversations","intercom_list_conversations_v2","intercom_list_tags_v2","intercom_open_conversation_v2","intercom_reply_conversation","intercom_reply_conversation_v2","intercom_search_contacts","intercom_search_contacts_v2","intercom_search_conversations","intercom_search_conversations_v2","intercom_snooze_conversation_v2","intercom_tag_contact_v2","intercom_tag_conversation_v2","intercom_untag_contact_v2","intercom_update_contact","intercom_update_contact_v2","intercom_update_ticket_v2","jina_read_url","jina_search","jira_add_attachment","jira_add_comment","jira_add_watcher","jira_add_worklog","jira_assign_issue","jira_bulk_read","jira_create_issue_link","jira_delete_attachment","jira_delete_comment","jira_delete_issue","jira_delete_issue_link","jira_delete_worklog","jira_get_attachments","jira_get_comments","jira_get_fields","jira_get_project","jira_get_transitions","jira_get_users","jira_get_worklogs","jira_list_issue_types","jira_list_projects","jira_remove_watcher","jira_retrieve","jira_search_issues","jira_search_users","jira_transition_issue","jira_update","jira_update_comment","jira_update_worklog","jira_write","jsm_add_comment","jsm_add_customer","jsm_add_organization","jsm_add_participants","jsm_answer_approval","jsm_attach_form","jsm_copy_forms","jsm_create_object","jsm_create_organization","jsm_create_request","jsm_delete_form","jsm_delete_object","jsm_externalise_form","jsm_get_approvals","jsm_get_comments","jsm_get_customers","jsm_get_form","jsm_get_form_answers","jsm_get_form_structure","jsm_get_form_templates","jsm_get_issue_forms","jsm_get_object","jsm_get_object_schema","jsm_get_object_type_attributes","jsm_get_organizations","jsm_get_participants","jsm_get_queues","jsm_get_request","jsm_get_request_type_fields","jsm_get_request_types","jsm_get_requests","jsm_get_service_desks","jsm_get_sla","jsm_get_transitions","jsm_internalise_form","jsm_list_object_schemas","jsm_list_object_types","jsm_reopen_form","jsm_save_form_answers","jsm_search_objects_aql","jsm_submit_form","jsm_transition_request","jsm_update_object","jupyter_copy_content","jupyter_create_file","jupyter_create_session","jupyter_delete_content","jupyter_delete_session","jupyter_get_content","jupyter_interrupt_kernel","jupyter_list_contents","jupyter_list_kernels","jupyter_list_kernelspecs","jupyter_list_sessions","jupyter_rename_content","jupyter_restart_kernel","jupyter_start_kernel","jupyter_stop_kernel","jupyter_upload_file","kalshi_amend_order","kalshi_amend_order_v2","kalshi_cancel_order","kalshi_cancel_order_v2","kalshi_create_order","kalshi_create_order_v2","kalshi_get_balance","kalshi_get_balance_v2","kalshi_get_candlesticks","kalshi_get_candlesticks_v2","kalshi_get_event","kalshi_get_event_candlesticks","kalshi_get_event_candlesticks_v2","kalshi_get_event_v2","kalshi_get_events","kalshi_get_events_v2","kalshi_get_exchange_announcements","kalshi_get_exchange_announcements_v2","kalshi_get_exchange_schedule","kalshi_get_exchange_schedule_v2","kalshi_get_exchange_status","kalshi_get_exchange_status_v2","kalshi_get_fills","kalshi_get_fills_v2","kalshi_get_market","kalshi_get_market_v2","kalshi_get_markets","kalshi_get_markets_v2","kalshi_get_order","kalshi_get_order_v2","kalshi_get_orderbook","kalshi_get_orderbook_v2","kalshi_get_orders","kalshi_get_orders_v2","kalshi_get_positions","kalshi_get_positions_v2","kalshi_get_series_by_ticker","kalshi_get_series_by_ticker_v2","kalshi_get_series_list","kalshi_get_series_list_v2","kalshi_get_settlements","kalshi_get_settlements_v2","kalshi_get_trades","kalshi_get_trades_v2","ketch_get_consent","ketch_get_subscriptions","ketch_invoke_right","ketch_set_consent","ketch_set_subscriptions","knowledge_create_document","knowledge_delete_chunk","knowledge_delete_document","knowledge_get_connector","knowledge_get_document","knowledge_list_chunks","knowledge_list_connectors","knowledge_list_documents","knowledge_list_tags","knowledge_search","knowledge_trigger_sync","knowledge_update_chunk","knowledge_upload_chunk","knowledge_upsert_document","langsmith_create_feedback","langsmith_create_run","langsmith_create_runs_batch","langsmith_get_run","langsmith_update_run","latex_compile","latex_get_package","latex_list_fonts","latex_search_packages","launchdarkly_create_flag","launchdarkly_delete_flag","launchdarkly_get_audit_log","launchdarkly_get_flag","launchdarkly_get_flag_status","launchdarkly_list_environments","launchdarkly_list_flags","launchdarkly_list_members","launchdarkly_list_projects","launchdarkly_list_segments","launchdarkly_toggle_flag","launchdarkly_update_flag","leadmagic_company_search","leadmagic_email_to_profile","leadmagic_find_email","leadmagic_find_mobile","leadmagic_get_credits","leadmagic_profile_search","leadmagic_profile_to_email","leadmagic_role_finder","leadmagic_validate_email","lemlist_get_activities","lemlist_get_lead","lemlist_send_email","linear_add_label_to_issue","linear_add_label_to_project","linear_archive_issue","linear_archive_label","linear_archive_project","linear_create_attachment","linear_create_comment","linear_create_customer","linear_create_customer_request","linear_create_customer_status","linear_create_customer_tier","linear_create_cycle","linear_create_favorite","linear_create_issue","linear_create_issue_relation","linear_create_label","linear_create_project","linear_create_project_label","linear_create_project_milestone","linear_create_project_status","linear_create_project_update","linear_create_workflow_state","linear_delete_attachment","linear_delete_comment","linear_delete_customer","linear_delete_customer_status","linear_delete_customer_tier","linear_delete_issue","linear_delete_issue_relation","linear_delete_project","linear_delete_project_label","linear_delete_project_milestone","linear_delete_project_status","linear_get_active_cycle","linear_get_customer","linear_get_cycle","linear_get_issue","linear_get_project","linear_get_viewer","linear_list_attachments","linear_list_comments","linear_list_customer_requests","linear_list_customer_statuses","linear_list_customer_tiers","linear_list_customers","linear_list_cycles","linear_list_favorites","linear_list_issue_relations","linear_list_labels","linear_list_notifications","linear_list_project_labels","linear_list_project_milestones","linear_list_project_statuses","linear_list_project_updates","linear_list_projects","linear_list_teams","linear_list_users","linear_list_workflow_states","linear_merge_customers","linear_read_issues","linear_remove_label_from_issue","linear_remove_label_from_project","linear_search_issues","linear_unarchive_issue","linear_update_attachment","linear_update_comment","linear_update_customer","linear_update_customer_request","linear_update_customer_status","linear_update_customer_tier","linear_update_issue","linear_update_label","linear_update_notification","linear_update_project","linear_update_project_label","linear_update_project_milestone","linear_update_project_status","linear_update_workflow_state","linkedin_get_profile","linkedin_share_post","linkup_search","linq_add_participant","linq_check_imessage","linq_check_rcs","linq_create_attachment","linq_create_chat","linq_create_contact_card","linq_create_webhook_subscription","linq_delete_attachment","linq_delete_message","linq_delete_webhook_subscription","linq_edit_message","linq_get_attachment","linq_get_chat","linq_get_contact_card","linq_get_message","linq_get_webhook_subscription","linq_leave_chat","linq_list_chats","linq_list_messages","linq_list_phone_numbers","linq_list_thread","linq_list_webhook_events","linq_list_webhook_subscriptions","linq_mark_chat_read","linq_react_to_message","linq_remove_participant","linq_send_message","linq_send_voice_memo","linq_share_contact_card","linq_start_typing","linq_stop_typing","linq_update_chat","linq_update_contact_card","linq_update_webhook_subscription","llm_chat","logfire_get_token_info","logfire_get_trace","logfire_query","logfire_search_records","logrocket_create_release","logrocket_get_audit_logs","logrocket_get_highlights","logrocket_identify_user","logrocket_list_exported_sessions","logrocket_request_highlights","logs_get","logs_get_execution","logs_get_run_details","logs_query","logs_query_runs","loops_check_contact_suppression","loops_create_contact","loops_create_contact_property","loops_delete_contact","loops_find_contact","loops_get_transactional_email","loops_list_contact_properties","loops_list_mailing_lists","loops_list_transactional_emails","loops_remove_contact_suppression","loops_send_event","loops_send_transactional_email","loops_update_contact","luma_add_guests","luma_cancel_event","luma_create_event","luma_get_event","luma_get_guest","luma_get_guests","luma_list_events","luma_lookup_event","luma_send_invites","luma_update_event","luma_update_guest_status","mailchimp_add_member","mailchimp_add_member_tags","mailchimp_add_or_update_member","mailchimp_add_segment_member","mailchimp_add_subscriber_to_automation","mailchimp_archive_member","mailchimp_create_audience","mailchimp_create_batch_operation","mailchimp_create_campaign","mailchimp_create_interest","mailchimp_create_interest_category","mailchimp_create_landing_page","mailchimp_create_merge_field","mailchimp_create_segment","mailchimp_create_template","mailchimp_delete_audience","mailchimp_delete_batch_operation","mailchimp_delete_campaign","mailchimp_delete_interest","mailchimp_delete_interest_category","mailchimp_delete_landing_page","mailchimp_delete_member","mailchimp_delete_merge_field","mailchimp_delete_segment","mailchimp_delete_template","mailchimp_get_audience","mailchimp_get_audiences","mailchimp_get_automation","mailchimp_get_automations","mailchimp_get_batch_operation","mailchimp_get_batch_operations","mailchimp_get_campaign","mailchimp_get_campaign_content","mailchimp_get_campaign_report","mailchimp_get_campaign_reports","mailchimp_get_campaigns","mailchimp_get_interest","mailchimp_get_interest_categories","mailchimp_get_interest_category","mailchimp_get_interests","mailchimp_get_landing_page","mailchimp_get_landing_pages","mailchimp_get_member","mailchimp_get_member_tags","mailchimp_get_members","mailchimp_get_merge_field","mailchimp_get_merge_fields","mailchimp_get_segment","mailchimp_get_segment_members","mailchimp_get_segments","mailchimp_get_template","mailchimp_get_templates","mailchimp_pause_automation","mailchimp_publish_landing_page","mailchimp_remove_member_tags","mailchimp_remove_segment_member","mailchimp_replicate_campaign","mailchimp_schedule_campaign","mailchimp_send_campaign","mailchimp_set_campaign_content","mailchimp_start_automation","mailchimp_unarchive_member","mailchimp_unpublish_landing_page","mailchimp_unschedule_campaign","mailchimp_update_audience","mailchimp_update_campaign","mailchimp_update_interest","mailchimp_update_interest_category","mailchimp_update_landing_page","mailchimp_update_member","mailchimp_update_merge_field","mailchimp_update_segment","mailchimp_update_template","mailgun_add_list_member","mailgun_create_mailing_list","mailgun_get_domain","mailgun_get_mailing_list","mailgun_get_message","mailgun_list_domains","mailgun_list_messages","mailgun_send_message","managed_agent_archive_session","managed_agent_create_session","managed_agent_delete_session","managed_agent_get_session","managed_agent_interrupt_session","managed_agent_list_events","managed_agent_respond_custom_tool","managed_agent_respond_tool_confirmation","managed_agent_run_session","managed_agent_send_message","managed_agent_update_session","mem0_add_memories","mem0_get_memories","mem0_search_memories","memory_add","memory_delete","memory_get","memory_get_all","microsoft_ad_add_directory_role_member","microsoft_ad_add_group_member","microsoft_ad_add_user_app_role_assignment","microsoft_ad_assign_license","microsoft_ad_create_group","microsoft_ad_create_user","microsoft_ad_delete_group","microsoft_ad_delete_user","microsoft_ad_get_conditional_access_policy","microsoft_ad_get_device","microsoft_ad_get_group","microsoft_ad_get_user","microsoft_ad_list_authentication_methods","microsoft_ad_list_conditional_access_policies","microsoft_ad_list_devices","microsoft_ad_list_directory_audits","microsoft_ad_list_directory_role_members","microsoft_ad_list_directory_roles","microsoft_ad_list_group_members","microsoft_ad_list_groups","microsoft_ad_list_service_principal_app_role_assignments","microsoft_ad_list_service_principals","microsoft_ad_list_sign_ins","microsoft_ad_list_subscribed_skus","microsoft_ad_list_user_app_role_assignments","microsoft_ad_list_user_devices","microsoft_ad_list_user_licenses","microsoft_ad_list_users","microsoft_ad_remove_directory_role_member","microsoft_ad_remove_group_member","microsoft_ad_remove_user_app_role_assignment","microsoft_ad_reset_password","microsoft_ad_revoke_sign_in_sessions","microsoft_ad_set_password","microsoft_ad_update_group","microsoft_ad_update_user","microsoft_dataverse_associate","microsoft_dataverse_create_multiple","microsoft_dataverse_create_record","microsoft_dataverse_delete_record","microsoft_dataverse_disassociate","microsoft_dataverse_download_file","microsoft_dataverse_execute_action","microsoft_dataverse_execute_function","microsoft_dataverse_fetchxml_query","microsoft_dataverse_get_entity_metadata","microsoft_dataverse_get_record","microsoft_dataverse_list_records","microsoft_dataverse_search","microsoft_dataverse_update_multiple","microsoft_dataverse_update_record","microsoft_dataverse_upload_file","microsoft_dataverse_upsert_record","microsoft_dataverse_whoami","microsoft_excel_clear_range","microsoft_excel_create_table","microsoft_excel_delete_worksheet","microsoft_excel_format_range","microsoft_excel_read","microsoft_excel_read_v2","microsoft_excel_sort_range","microsoft_excel_table_add","microsoft_excel_worksheet_add","microsoft_excel_write","microsoft_excel_write_v2","microsoft_planner_create_bucket","microsoft_planner_create_plan","microsoft_planner_create_task","microsoft_planner_delete_bucket","microsoft_planner_delete_plan","microsoft_planner_delete_task","microsoft_planner_get_plan_details","microsoft_planner_get_task_details","microsoft_planner_list_buckets","microsoft_planner_list_plans","microsoft_planner_read_bucket","microsoft_planner_read_plan","microsoft_planner_read_task","microsoft_planner_update_bucket","microsoft_planner_update_plan","microsoft_planner_update_plan_details","microsoft_planner_update_task","microsoft_planner_update_task_details","microsoft_teams_delete_channel_message","microsoft_teams_delete_chat_message","microsoft_teams_get_message","microsoft_teams_list_channel_members","microsoft_teams_list_channels","microsoft_teams_list_chat_members","microsoft_teams_list_chats","microsoft_teams_list_team_members","microsoft_teams_list_teams","microsoft_teams_read_channel","microsoft_teams_read_chat","microsoft_teams_reply_to_message","microsoft_teams_set_reaction","microsoft_teams_unset_reaction","microsoft_teams_update_channel_message","microsoft_teams_update_chat_message","microsoft_teams_write_channel","microsoft_teams_write_chat","millionverifier_get_credits","millionverifier_verify_email","mintlify_create_agent_job","mintlify_create_assistant_message","mintlify_detect_ai_prose","mintlify_get_agent_job","mintlify_get_assistant_caller_stats","mintlify_get_assistant_conversations","mintlify_get_feedback","mintlify_get_feedback_by_page","mintlify_get_page_content","mintlify_get_searches","mintlify_get_update_status","mintlify_get_views","mintlify_get_visitors","mintlify_search","mintlify_send_agent_message","mintlify_trigger_automation","mintlify_trigger_preview","mintlify_trigger_update","mistral_parser","mistral_parser_v2","mistral_parser_v3","monday_archive_item","monday_change_column_value","monday_create_board","monday_create_column","monday_create_group","monday_create_item","monday_create_subitem","monday_create_update","monday_delete_item","monday_duplicate_item","monday_get_board","monday_get_groups","monday_get_item","monday_get_items","monday_list_boards","monday_move_item_to_group","monday_search_items","monday_update_item","mongodb_delete","mongodb_execute","mongodb_insert","mongodb_introspect","mongodb_query","mongodb_update","mssql_delete","mssql_execute","mssql_insert","mssql_introspect","mssql_query","mssql_update","mysql_delete","mysql_execute","mysql_insert","mysql_introspect","mysql_query","mysql_update","neo4j_create","neo4j_delete","neo4j_execute","neo4j_introspect","neo4j_merge","neo4j_query","neo4j_update","netsuite_attach_record","netsuite_batch_create_records","netsuite_batch_delete_records","netsuite_batch_get_records","netsuite_batch_update_records","netsuite_batch_upsert_records","netsuite_create_record","netsuite_delete_record","netsuite_detach_record","netsuite_execute_action","netsuite_execute_dataset","netsuite_execute_suiteql","netsuite_get_async_result","netsuite_get_async_status","netsuite_get_governance_limits","netsuite_get_record","netsuite_get_record_form","netsuite_get_record_metadata","netsuite_get_select_options","netsuite_get_server_time","netsuite_get_subresource","netsuite_list_datasets","netsuite_list_record_types","netsuite_list_records","netsuite_transform_record","netsuite_update_record","netsuite_upsert_record","neverbounce_get_credits","neverbounce_verify_email","new_relic_create_deployment_event","new_relic_get_entity","new_relic_nrql_query","new_relic_search_entities","notion_add_database_row","notion_add_database_row_v2","notion_append_blocks","notion_append_blocks_v2","notion_create_comment","notion_create_comment_v2","notion_create_database","notion_create_database_v2","notion_create_page","notion_create_page_v2","notion_delete_block","notion_delete_block_v2","notion_list_comments","notion_list_comments_v2","notion_list_users","notion_list_users_v2","notion_query_database","notion_query_database_v2","notion_read","notion_read_database","notion_read_database_v2","notion_read_v2","notion_retrieve_block","notion_retrieve_block_children","notion_retrieve_block_children_v2","notion_retrieve_block_v2","notion_retrieve_user","notion_retrieve_user_v2","notion_search","notion_search_v2","notion_update_block","notion_update_block_v2","notion_update_page","notion_update_page_v2","notion_write","notion_write_v2","obsidian_append_active","obsidian_append_note","obsidian_append_periodic_note","obsidian_create_note","obsidian_delete_note","obsidian_execute_command","obsidian_get_active","obsidian_get_note","obsidian_get_periodic_note","obsidian_list_commands","obsidian_list_files","obsidian_open_file","obsidian_patch_active","obsidian_patch_note","obsidian_search","okta_activate_group_rule","okta_activate_user","okta_add_user_to_group","okta_assign_group_to_app","okta_assign_user_role","okta_assign_user_to_app","okta_clear_user_sessions","okta_create_group","okta_create_group_rule","okta_create_user","okta_deactivate_group_rule","okta_deactivate_user","okta_delete_group","okta_delete_group_rule","okta_delete_user","okta_enroll_factor","okta_get_app","okta_get_factor","okta_get_group","okta_get_group_rule","okta_get_logs","okta_get_session","okta_get_user","okta_list_app_groups","okta_list_app_users","okta_list_apps","okta_list_factors","okta_list_group_members","okta_list_group_rules","okta_list_groups","okta_list_user_roles","okta_list_users","okta_remove_group_from_app","okta_remove_user_from_app","okta_remove_user_from_group","okta_remove_user_role","okta_reset_all_factors","okta_reset_factor","okta_reset_password","okta_revoke_session","okta_suspend_user","okta_unsuspend_user","okta_update_group","okta_update_user","onedrive_copy","onedrive_create_folder","onedrive_create_share_link","onedrive_delete","onedrive_download","onedrive_get_drive_info","onedrive_get_item","onedrive_list","onedrive_move","onedrive_search","onedrive_upload","onepassword_create_item","onepassword_delete_item","onepassword_get_item","onepassword_get_item_file","onepassword_get_vault","onepassword_list_items","onepassword_list_vaults","onepassword_replace_item","onepassword_resolve_secret","onepassword_update_item","openai_embeddings","openai_image","outlook_calendar_create_event","outlook_calendar_delete_event","outlook_calendar_get_event","outlook_calendar_list_events","outlook_calendar_respond","outlook_calendar_update_event","outlook_copy","outlook_create_folder","outlook_delete","outlook_draft","outlook_forward","outlook_get_attachment","outlook_list_attachments","outlook_list_folders","outlook_mark_read","outlook_mark_unread","outlook_move","outlook_read","outlook_reply","outlook_reply_all","outlook_search","outlook_send","outlook_update_message","pagerduty_add_note","pagerduty_create_incident","pagerduty_get_incident","pagerduty_get_service","pagerduty_list_escalation_policies","pagerduty_list_incident_alerts","pagerduty_list_incidents","pagerduty_list_oncalls","pagerduty_list_schedules","pagerduty_list_services","pagerduty_list_users","pagerduty_merge_incidents","pagerduty_send_event","pagerduty_snooze_incident","pagerduty_update_incident","parallel_deep_research","parallel_extract","parallel_search","pdl_autocomplete","pdl_bulk_company_enrich","pdl_bulk_person_enrich","pdl_clean_company","pdl_clean_location","pdl_clean_school","pdl_company_enrich","pdl_company_search","pdl_person_enrich","pdl_person_identify","pdl_person_search","perplexity_chat","perplexity_search","persona_approve_inquiry","persona_create_account","persona_create_inquiry","persona_create_report","persona_decline_inquiry","persona_expire_inquiry","persona_generate_inquiry_link","persona_get_account","persona_get_case","persona_get_document","persona_get_inquiry","persona_get_report","persona_get_verification","persona_import_accounts","persona_list_accounts","persona_list_cases","persona_list_inquiries","persona_list_inquiry_templates","persona_list_reports","persona_mark_inquiry_for_review","persona_print_inquiry_pdf","persona_redact_account","persona_redact_inquiry","persona_resume_inquiry","persona_update_account","persona_update_inquiry","pinecone_delete_vectors","pinecone_describe_index","pinecone_describe_index_stats","pinecone_fetch","pinecone_generate_embeddings","pinecone_list_indexes","pinecone_list_vector_ids","pinecone_search_text","pinecone_search_vector","pinecone_update_vector","pinecone_upsert_text","pipedrive_create_activity","pipedrive_create_deal","pipedrive_create_lead","pipedrive_create_project","pipedrive_delete_lead","pipedrive_get_activities","pipedrive_get_all_deals","pipedrive_get_deal","pipedrive_get_files","pipedrive_get_leads","pipedrive_get_mail_messages","pipedrive_get_mail_thread","pipedrive_get_pipeline_deals","pipedrive_get_pipelines","pipedrive_get_projects","pipedrive_update_activity","pipedrive_update_deal","pipedrive_update_lead","polymarket_get_activity","polymarket_get_event","polymarket_get_events","polymarket_get_holders","polymarket_get_last_trade_price","polymarket_get_leaderboard","polymarket_get_market","polymarket_get_markets","polymarket_get_midpoint","polymarket_get_orderbook","polymarket_get_positions","polymarket_get_price","polymarket_get_price_history","polymarket_get_series","polymarket_get_series_by_id","polymarket_get_spread","polymarket_get_tags","polymarket_get_tick_size","polymarket_get_trades","polymarket_search","postgresql_delete","postgresql_execute","postgresql_insert","postgresql_introspect","postgresql_query","postgresql_update","posthog_batch_events","posthog_capture_event","posthog_create_annotation","posthog_create_cohort","posthog_create_dashboard","posthog_create_experiment","posthog_create_feature_flag","posthog_create_insight","posthog_create_survey","posthog_delete_feature_flag","posthog_delete_person","posthog_delete_survey","posthog_evaluate_flags","posthog_get_cohort","posthog_get_dashboard","posthog_get_event_definition","posthog_get_experiment","posthog_get_feature_flag","posthog_get_insight","posthog_get_organization","posthog_get_person","posthog_get_project","posthog_get_property_definition","posthog_get_session_recording","posthog_get_survey","posthog_list_actions","posthog_list_annotations","posthog_list_cohorts","posthog_list_dashboards","posthog_list_event_definitions","posthog_list_experiments","posthog_list_feature_flags","posthog_list_insights","posthog_list_organizations","posthog_list_persons","posthog_list_projects","posthog_list_property_definitions","posthog_list_recording_playlists","posthog_list_session_recordings","posthog_list_surveys","posthog_query","posthog_update_cohort","posthog_update_event_definition","posthog_update_experiment","posthog_update_feature_flag","posthog_update_insight","posthog_update_property_definition","posthog_update_survey","profound_bot_logs","profound_bots_report","profound_category_assets","profound_category_personas","profound_category_prompts","profound_category_tags","profound_category_topics","profound_citation_prompts","profound_citations_report","profound_list_assets","profound_list_categories","profound_list_domains","profound_list_models","profound_list_optimizations","profound_list_personas","profound_list_regions","profound_optimization_analysis","profound_prompt_answers","profound_prompt_volume","profound_query_fanouts","profound_raw_logs","profound_referrals_report","profound_sentiment_report","profound_visibility_report","prospeo_account_information","prospeo_bulk_enrich_company","prospeo_bulk_enrich_person","prospeo_enrich_company","prospeo_enrich_person","prospeo_search_company","prospeo_search_person","prospeo_search_suggestions","pulse_parser","pulse_parser_v2","qdrant_fetch_points","qdrant_search_vector","qdrant_upsert_points","quartr_get_audio","quartr_get_company","quartr_get_event","quartr_get_event_summary","quartr_get_report","quartr_get_slide_deck","quartr_get_transcript","quartr_list_audio","quartr_list_companies","quartr_list_document_types","quartr_list_documents","quartr_list_event_types","quartr_list_events","quartr_list_live_events","quartr_list_reports","quartr_list_slide_decks","quartr_list_transcripts","quiver_image_to_svg","quiver_list_models","quiver_text_to_svg","rabbitmq_create_binding","rabbitmq_create_exchange","rabbitmq_create_policy","rabbitmq_create_queue","rabbitmq_delete_binding","rabbitmq_delete_exchange","rabbitmq_delete_policy","rabbitmq_delete_queue","rabbitmq_get_exchange","rabbitmq_get_messages","rabbitmq_get_overview","rabbitmq_get_queue","rabbitmq_health_check","rabbitmq_list_bindings","rabbitmq_list_channels","rabbitmq_list_connections","rabbitmq_list_consumers","rabbitmq_list_exchange_bindings","rabbitmq_list_exchanges","rabbitmq_list_nodes","rabbitmq_list_policies","rabbitmq_list_queues","rabbitmq_list_vhosts","rabbitmq_publish_message","rabbitmq_purge_queue","railway_create_environment","railway_create_project","railway_create_service","railway_delete_environment","railway_delete_project","railway_delete_service","railway_delete_variable","railway_deploy_service","railway_get_deployment","railway_get_deployment_logs","railway_get_project","railway_list_deployments","railway_list_project_members","railway_list_projects","railway_list_variables","railway_restart_deployment","railway_rollback_deployment","railway_transfer_project","railway_update_project","railway_upsert_variable","rb2b_credit_check","rb2b_email_to_activity","rb2b_hem_to_best_linkedin","rb2b_hem_to_business_profile","rb2b_hem_to_linkedin","rb2b_hem_to_maid","rb2b_ip_to_company","rb2b_ip_to_hem","rb2b_ip_to_maid","rb2b_linkedin_slug_search","rb2b_linkedin_to_best_personal_email","rb2b_linkedin_to_business_profile","rb2b_linkedin_to_hashed_emails","rb2b_linkedin_to_mobile_phone","rb2b_linkedin_to_personal_email","rds_delete","rds_execute","rds_insert","rds_introspect","rds_query","rds_update","reddit_delete","reddit_edit","reddit_get_comments","reddit_get_controversial","reddit_get_info","reddit_get_me","reddit_get_messages","reddit_get_posts","reddit_get_saved","reddit_get_subreddit_info","reddit_get_subreddit_rules","reddit_get_user","reddit_get_user_comments","reddit_get_user_posts","reddit_hide","reddit_hot_posts","reddit_list_my_subreddits","reddit_lock","reddit_mark_all_read","reddit_mark_read","reddit_marknsfw","reddit_mod_approve","reddit_mod_distinguish","reddit_mod_remove","reddit_mod_sticky","reddit_reply","reddit_report","reddit_save","reddit_search","reddit_search_subreddits","reddit_send_message","reddit_submit_post","reddit_subscribe","reddit_unhide","reddit_unlock","reddit_unmarknsfw","reddit_unsave","reddit_vote","redis_command","redis_delete","redis_exists","redis_expire","redis_get","redis_hdel","redis_hget","redis_hgetall","redis_hset","redis_incr","redis_incrby","redis_keys","redis_llen","redis_lpop","redis_lpush","redis_lrange","redis_persist","redis_rpop","redis_rpush","redis_set","redis_setnx","redis_ttl","reducto_parser","reducto_parser_v2","resend_cancel_email","resend_create_audience","resend_create_broadcast","resend_create_contact","resend_delete_audience","resend_delete_contact","resend_get_audience","resend_get_broadcast","resend_get_contact","resend_get_email","resend_list_audiences","resend_list_contacts","resend_list_domains","resend_send","resend_send_broadcast","resend_update_contact","revenuecat_create_purchase","revenuecat_defer_google_subscription","revenuecat_delete_customer","revenuecat_get_customer","revenuecat_grant_entitlement","revenuecat_list_offerings","revenuecat_refund_google_subscription","revenuecat_revoke_entitlement","revenuecat_revoke_google_subscription","revenuecat_update_subscriber_attributes","rippling_bulk_create_custom_object_records","rippling_bulk_delete_custom_object_records","rippling_bulk_update_custom_object_records","rippling_create_business_partner","rippling_create_business_partner_group","rippling_create_custom_app","rippling_create_custom_object","rippling_create_custom_object_field","rippling_create_custom_object_record","rippling_create_custom_page","rippling_create_custom_setting","rippling_create_department","rippling_create_draft_hires","rippling_create_object_category","rippling_create_title","rippling_create_work_location","rippling_delete_business_partner","rippling_delete_business_partner_group","rippling_delete_custom_app","rippling_delete_custom_object","rippling_delete_custom_object_field","rippling_delete_custom_object_record","rippling_delete_custom_page","rippling_delete_custom_setting","rippling_delete_object_category","rippling_delete_title","rippling_delete_work_location","rippling_get_business_partner","rippling_get_business_partner_group","rippling_get_current_user","rippling_get_custom_app","rippling_get_custom_object","rippling_get_custom_object_field","rippling_get_custom_object_record","rippling_get_custom_object_record_by_external_id","rippling_get_custom_page","rippling_get_custom_setting","rippling_get_department","rippling_get_employment_type","rippling_get_job_function","rippling_get_object_category","rippling_get_report_run","rippling_get_supergroup","rippling_get_team","rippling_get_title","rippling_get_user","rippling_get_work_location","rippling_get_worker","rippling_list_business_partner_groups","rippling_list_business_partners","rippling_list_companies","rippling_list_custom_apps","rippling_list_custom_fields","rippling_list_custom_object_fields","rippling_list_custom_object_records","rippling_list_custom_objects","rippling_list_custom_pages","rippling_list_custom_settings","rippling_list_departments","rippling_list_employment_types","rippling_list_entitlements","rippling_list_job_functions","rippling_list_object_categories","rippling_list_supergroup_exclusion_members","rippling_list_supergroup_inclusion_members","rippling_list_supergroup_members","rippling_list_supergroups","rippling_list_teams","rippling_list_titles","rippling_list_users","rippling_list_work_locations","rippling_list_workers","rippling_query_custom_object_records","rippling_trigger_report_run","rippling_update_custom_app","rippling_update_custom_object","rippling_update_custom_object_field","rippling_update_custom_object_record","rippling_update_custom_page","rippling_update_custom_setting","rippling_update_department","rippling_update_object_category","rippling_update_supergroup_exclusion_members","rippling_update_supergroup_inclusion_members","rippling_update_title","rippling_update_work_location","rocketlane_add_field_option","rocketlane_add_project_members","rocketlane_add_task_assignees","rocketlane_add_task_dependencies","rocketlane_add_task_followers","rocketlane_archive_project","rocketlane_assign_placeholders","rocketlane_create_field","rocketlane_create_phase","rocketlane_create_project","rocketlane_create_space","rocketlane_create_space_document","rocketlane_create_task","rocketlane_create_time_entry","rocketlane_create_time_off","rocketlane_delete_field","rocketlane_delete_phase","rocketlane_delete_project","rocketlane_delete_space","rocketlane_delete_space_document","rocketlane_delete_task","rocketlane_delete_time_entry","rocketlane_delete_time_off","rocketlane_get_field","rocketlane_get_invoice","rocketlane_get_invoice_line_items","rocketlane_get_invoice_payments","rocketlane_get_phase","rocketlane_get_project","rocketlane_get_space","rocketlane_get_space_document","rocketlane_get_task","rocketlane_get_time_entry","rocketlane_get_time_off","rocketlane_get_user","rocketlane_import_template","rocketlane_list_fields","rocketlane_list_invoices","rocketlane_list_phases","rocketlane_list_placeholders","rocketlane_list_projects","rocketlane_list_resource_allocations","rocketlane_list_space_documents","rocketlane_list_spaces","rocketlane_list_tasks","rocketlane_list_time_entries","rocketlane_list_time_entry_categories","rocketlane_list_time_offs","rocketlane_list_users","rocketlane_move_task_to_phase","rocketlane_remove_project_members","rocketlane_remove_task_assignees","rocketlane_remove_task_dependencies","rocketlane_remove_task_followers","rocketlane_search_time_entries","rocketlane_unassign_placeholders","rocketlane_update_field","rocketlane_update_field_option","rocketlane_update_phase","rocketlane_update_project","rocketlane_update_space","rocketlane_update_space_document","rocketlane_update_task","rocketlane_update_time_entry","rootly_acknowledge_alert","rootly_add_incident_event","rootly_add_subscribers","rootly_assign_incident_role","rootly_create_action_item","rootly_create_alert","rootly_create_incident","rootly_create_status_page_event","rootly_delete_action_item","rootly_delete_incident","rootly_escalate_alert","rootly_get_alert","rootly_get_incident","rootly_list_action_items","rootly_list_alerts","rootly_list_causes","rootly_list_environments","rootly_list_escalation_policies","rootly_list_functionalities","rootly_list_incident_events","rootly_list_incident_roles","rootly_list_incident_types","rootly_list_incidents","rootly_list_on_calls","rootly_list_playbooks","rootly_list_retrospectives","rootly_list_schedules","rootly_list_services","rootly_list_severities","rootly_list_teams","rootly_list_users","rootly_mitigate_incident","rootly_remove_subscribers","rootly_resolve_alert","rootly_resolve_incident","rootly_run_workflow","rootly_snooze_alert","rootly_unassign_incident_role","rootly_update_action_item","rootly_update_alert","rootly_update_incident","s3_copy_object","s3_create_bucket","s3_delete_bucket","s3_delete_object","s3_delete_objects","s3_get_object","s3_head_object","s3_list_buckets","s3_list_objects","s3_presigned_url","s3_put_object","salesforce_create_account","salesforce_create_case","salesforce_create_contact","salesforce_create_custom_field","salesforce_create_custom_object","salesforce_create_lead","salesforce_create_opportunity","salesforce_create_task","salesforce_delete_account","salesforce_delete_case","salesforce_delete_contact","salesforce_delete_custom_field","salesforce_delete_lead","salesforce_delete_opportunity","salesforce_delete_task","salesforce_describe_object","salesforce_get_accounts","salesforce_get_cases","salesforce_get_contacts","salesforce_get_dashboard","salesforce_get_leads","salesforce_get_opportunities","salesforce_get_report","salesforce_get_tasks","salesforce_list_dashboards","salesforce_list_objects","salesforce_list_report_types","salesforce_list_reports","salesforce_query","salesforce_query_more","salesforce_refresh_dashboard","salesforce_run_report","salesforce_tooling_query","salesforce_update_account","salesforce_update_case","salesforce_update_contact","salesforce_update_custom_field","salesforce_update_lead","salesforce_update_opportunity","salesforce_update_task","sap_concur_approve_expense_report","sap_concur_associate_attendees","sap_concur_create_cash_advance","sap_concur_create_expected_expense","sap_concur_create_expense_report","sap_concur_create_list_item","sap_concur_create_purchase_request","sap_concur_create_quick_expense","sap_concur_create_quick_expense_with_image","sap_concur_create_report_comment","sap_concur_create_travel_request","sap_concur_create_user","sap_concur_delete_expected_expense","sap_concur_delete_expense","sap_concur_delete_expense_report","sap_concur_delete_list_item","sap_concur_delete_travel_request","sap_concur_delete_user","sap_concur_get_allocation","sap_concur_get_budget","sap_concur_get_cash_advance","sap_concur_get_expected_expense","sap_concur_get_expense","sap_concur_get_expense_report","sap_concur_get_itemizations","sap_concur_get_itinerary","sap_concur_get_list","sap_concur_get_list_item","sap_concur_get_purchase_request","sap_concur_get_receipt","sap_concur_get_receipt_status","sap_concur_get_request_cash_advance","sap_concur_get_travel_profile","sap_concur_get_travel_request","sap_concur_get_user","sap_concur_issue_cash_advance","sap_concur_list_allocations","sap_concur_list_attendee_associations","sap_concur_list_budget_categories","sap_concur_list_budgets","sap_concur_list_exceptions","sap_concur_list_expected_expenses","sap_concur_list_expense_reports","sap_concur_list_expenses","sap_concur_list_itineraries","sap_concur_list_list_items","sap_concur_list_lists","sap_concur_list_receipts","sap_concur_list_report_comments","sap_concur_list_reports_to_approve","sap_concur_list_travel_profiles_summary","sap_concur_list_travel_request_comments","sap_concur_list_travel_requests","sap_concur_list_users","sap_concur_move_travel_request","sap_concur_recall_expense_report","sap_concur_remove_all_attendees","sap_concur_search_locations","sap_concur_search_users","sap_concur_send_back_expense_report","sap_concur_submit_expense_report","sap_concur_update_allocation","sap_concur_update_expected_expense","sap_concur_update_expense","sap_concur_update_expense_report","sap_concur_update_list_item","sap_concur_update_travel_request","sap_concur_update_user","sap_concur_upload_exchange_rates","sap_concur_upload_receipt_image","sap_s4hana_create_business_partner","sap_s4hana_create_purchase_order","sap_s4hana_create_purchase_requisition","sap_s4hana_create_sales_order","sap_s4hana_delete_sales_order","sap_s4hana_get_billing_document","sap_s4hana_get_business_partner","sap_s4hana_get_customer","sap_s4hana_get_inbound_delivery","sap_s4hana_get_material_document","sap_s4hana_get_outbound_delivery","sap_s4hana_get_product","sap_s4hana_get_purchase_order","sap_s4hana_get_purchase_requisition","sap_s4hana_get_sales_order","sap_s4hana_get_supplier","sap_s4hana_get_supplier_invoice","sap_s4hana_list_billing_documents","sap_s4hana_list_business_partners","sap_s4hana_list_customers","sap_s4hana_list_inbound_deliveries","sap_s4hana_list_material_documents","sap_s4hana_list_material_stock","sap_s4hana_list_outbound_deliveries","sap_s4hana_list_products","sap_s4hana_list_purchase_orders","sap_s4hana_list_purchase_requisitions","sap_s4hana_list_sales_orders","sap_s4hana_list_supplier_invoices","sap_s4hana_list_suppliers","sap_s4hana_odata_query","sap_s4hana_update_business_partner","sap_s4hana_update_customer","sap_s4hana_update_product","sap_s4hana_update_purchase_order","sap_s4hana_update_purchase_requisition","sap_s4hana_update_sales_order","sap_s4hana_update_supplier","search_tool","secrets_manager_create_secret","secrets_manager_delete_secret","secrets_manager_describe_secret","secrets_manager_get_secret","secrets_manager_list_secrets","secrets_manager_restore_secret","secrets_manager_rotate_secret","secrets_manager_tag_resource","secrets_manager_untag_resource","secrets_manager_update_secret","sendblue_evaluate_service","sendblue_get_message","sendblue_send_group_message","sendblue_send_message","sendblue_send_typing_indicator","sendgrid_add_contact","sendgrid_add_contacts_to_list","sendgrid_create_list","sendgrid_create_template","sendgrid_create_template_version","sendgrid_delete_contacts","sendgrid_delete_list","sendgrid_delete_template","sendgrid_get_contact","sendgrid_get_list","sendgrid_get_template","sendgrid_list_all_lists","sendgrid_list_templates","sendgrid_remove_contacts_from_list","sendgrid_search_contacts","sendgrid_send_mail","sentry_events_get","sentry_events_list","sentry_issues_get","sentry_issues_list","sentry_issues_update","sentry_projects_create","sentry_projects_get","sentry_projects_list","sentry_projects_update","sentry_releases_create","sentry_releases_deploy","sentry_releases_list","sentry_teams_list","serper_search","servicenow_add_incident_comment","servicenow_aggregate","servicenow_close_incident","servicenow_create_change_request","servicenow_create_incident","servicenow_create_record","servicenow_delete_record","servicenow_download_attachment","servicenow_find_user","servicenow_get_change_next_states","servicenow_get_change_request","servicenow_get_ci","servicenow_get_incident","servicenow_get_knowledge_article","servicenow_get_requested_item","servicenow_list_approvals","servicenow_list_attachments","servicenow_list_catalog_items","servicenow_list_change_requests","servicenow_list_change_tasks","servicenow_list_ci_relationships","servicenow_list_group_members","servicenow_list_incidents","servicenow_list_requested_items","servicenow_order_catalog_item","servicenow_read_record","servicenow_resolve_incident","servicenow_search_cis","servicenow_search_knowledge","servicenow_update_approval","servicenow_update_change_request","servicenow_update_change_state","servicenow_update_incident","servicenow_update_record","servicenow_upload_attachment","ses_create_configuration_set","ses_create_email_identity","ses_create_template","ses_delete_email_identity","ses_delete_suppressed_destination","ses_delete_template","ses_get_account","ses_get_email_identity","ses_get_suppressed_destination","ses_get_template","ses_list_identities","ses_list_suppressed_destinations","ses_list_templates","ses_put_suppressed_destination","ses_send_bulk_email","ses_send_custom_verification_email","ses_send_email","ses_send_templated_email","ses_update_template","sftp_delete","sftp_download","sftp_list","sftp_mkdir","sftp_upload","sharepoint_add_list_items","sharepoint_create_list","sharepoint_create_page","sharepoint_delete_file","sharepoint_delete_list_item","sharepoint_delete_page","sharepoint_download_file","sharepoint_get_drive_item","sharepoint_get_list","sharepoint_get_list_item","sharepoint_list_sites","sharepoint_publish_page","sharepoint_read_page","sharepoint_update_list","sharepoint_update_page","sharepoint_upload_file","shopify_adjust_inventory","shopify_cancel_order","shopify_create_customer","shopify_create_fulfillment","shopify_create_product","shopify_delete_customer","shopify_delete_product","shopify_get_collection","shopify_get_customer","shopify_get_inventory_level","shopify_get_order","shopify_get_product","shopify_list_collections","shopify_list_customers","shopify_list_inventory_items","shopify_list_locations","shopify_list_orders","shopify_list_products","shopify_update_customer","shopify_update_order","shopify_update_product","similarweb_bounce_rate","similarweb_page_views","similarweb_pages_per_visit","similarweb_traffic_visits","similarweb_visit_duration","similarweb_website_overview","sixtyfour_enrich_company","sixtyfour_enrich_lead","sixtyfour_find_email","sixtyfour_find_phone","slack_add_reaction","slack_archive_conversation","slack_canvas","slack_create_channel_canvas","slack_create_conversation","slack_delete_canvas","slack_delete_message","slack_delete_scheduled_message","slack_download","slack_edit_canvas","slack_ephemeral_message","slack_get_canvas","slack_get_channel_history","slack_get_channel_info","slack_get_message","slack_get_permalink","slack_get_thread","slack_get_thread_replies","slack_get_user","slack_get_user_presence","slack_invite_to_conversation","slack_list_canvases","slack_list_channels","slack_list_members","slack_list_scheduled_messages","slack_list_users","slack_lookup_canvas_sections","slack_message","slack_message_reader","slack_open_view","slack_publish_view","slack_push_view","slack_remove_reaction","slack_rename_conversation","slack_schedule_message","slack_set_conversation_purpose","slack_set_conversation_topic","slack_set_status","slack_set_suggested_prompts","slack_set_title","slack_update_message","slack_update_view","smartlead_add_email_accounts_to_campaign","smartlead_add_leads_to_campaign","smartlead_create_campaign","smartlead_create_lead_list","smartlead_delete_campaign","smartlead_delete_campaign_webhook","smartlead_delete_lead_from_campaign","smartlead_delete_lead_list","smartlead_duplicate_campaign","smartlead_export_campaign_leads","smartlead_get_campaign","smartlead_get_campaign_analytics","smartlead_get_campaign_analytics_by_date","smartlead_get_campaign_lead_statistics","smartlead_get_campaign_mailbox_statistics","smartlead_get_campaign_sequences","smartlead_get_campaign_statistics","smartlead_get_campaign_top_level_analytics_by_date","smartlead_get_campaign_webhook_summary","smartlead_get_lead_by_email","smartlead_get_lead_by_id","smartlead_get_lead_list","smartlead_get_lead_message_history","smartlead_list_campaign_email_accounts","smartlead_list_campaign_leads","smartlead_list_campaign_webhooks","smartlead_list_campaigns","smartlead_list_clients","smartlead_list_email_accounts","smartlead_list_inbox_replies","smartlead_list_lead_activities","smartlead_list_lead_categories","smartlead_list_lead_lists","smartlead_mark_lead_complete","smartlead_pause_lead","smartlead_remove_email_accounts_from_campaign","smartlead_resume_lead","smartlead_save_campaign_sequences","smartlead_unsubscribe_lead_from_campaign","smartlead_unsubscribe_lead_globally","smartlead_update_campaign_schedule","smartlead_update_campaign_settings","smartlead_update_campaign_status","smartlead_update_lead","smartlead_update_lead_category","smartlead_update_lead_list","smartlead_upsert_campaign_webhook","sms_send","smtp_send_mail","snowflake_alter_warehouse","snowflake_call_procedure","snowflake_cancel_statement","snowflake_cancel_task_run","snowflake_delete_rows","snowflake_execute_sql","snowflake_get_statement","snowflake_get_task","snowflake_get_task_run","snowflake_get_task_run_output","snowflake_get_warehouse","snowflake_insert_rows","snowflake_introspect_schema","snowflake_list_copy_history","snowflake_list_databases","snowflake_list_query_history","snowflake_list_schemas","snowflake_list_tables","snowflake_list_task_runs","snowflake_list_tasks","snowflake_list_warehouses","snowflake_load_data","snowflake_resume_task","snowflake_resume_warehouse","snowflake_run_task","snowflake_suspend_task","snowflake_suspend_warehouse","snowflake_unload_data","snowflake_update_rows","snowflake_upsert_rows","splunk_cancel_search_job","splunk_create_search_job","splunk_dispatch_saved_search","splunk_get_fired_alerts","splunk_get_saved_search","splunk_get_search_job","splunk_get_search_results","splunk_list_apps","splunk_list_fired_alerts","splunk_list_indexes","splunk_list_saved_searches","splunk_run_search","sportmonks_core_get_cities","sportmonks_core_get_city","sportmonks_core_get_continent","sportmonks_core_get_continents","sportmonks_core_get_countries","sportmonks_core_get_country","sportmonks_core_get_entity_filters","sportmonks_core_get_my_usage","sportmonks_core_get_region","sportmonks_core_get_regions","sportmonks_core_get_timezones","sportmonks_core_get_type","sportmonks_core_get_type_by_entity","sportmonks_core_get_types","sportmonks_core_search_cities","sportmonks_core_search_countries","sportmonks_core_search_regions","sportmonks_football_expected_by_player","sportmonks_football_expected_by_team","sportmonks_football_get_all_commentaries","sportmonks_football_get_all_fixtures","sportmonks_football_get_all_players","sportmonks_football_get_all_rivals","sportmonks_football_get_all_teams","sportmonks_football_get_all_transfer_rumours","sportmonks_football_get_all_transfers","sportmonks_football_get_brackets_by_season","sportmonks_football_get_coach","sportmonks_football_get_coaches","sportmonks_football_get_coaches_by_country","sportmonks_football_get_commentaries_by_fixture","sportmonks_football_get_current_leagues_by_team","sportmonks_football_get_expected_lineups_by_player","sportmonks_football_get_expected_lineups_by_team","sportmonks_football_get_extended_team_squad","sportmonks_football_get_fixture","sportmonks_football_get_fixtures_by_date","sportmonks_football_get_fixtures_by_date_range","sportmonks_football_get_fixtures_by_date_range_for_team","sportmonks_football_get_fixtures_by_ids","sportmonks_football_get_grouped_standings_by_round","sportmonks_football_get_head_to_head","sportmonks_football_get_inplay_livescores","sportmonks_football_get_latest_coaches","sportmonks_football_get_latest_fixtures","sportmonks_football_get_latest_livescores","sportmonks_football_get_latest_players","sportmonks_football_get_latest_totw","sportmonks_football_get_latest_transfers","sportmonks_football_get_league","sportmonks_football_get_leagues","sportmonks_football_get_leagues_by_country","sportmonks_football_get_leagues_by_date","sportmonks_football_get_leagues_by_team","sportmonks_football_get_live_leagues","sportmonks_football_get_live_probabilities","sportmonks_football_get_live_probabilities_by_fixture","sportmonks_football_get_live_standings_by_league","sportmonks_football_get_livescores","sportmonks_football_get_match_facts","sportmonks_football_get_match_facts_by_date_range","sportmonks_football_get_match_facts_by_fixture","sportmonks_football_get_match_facts_by_league","sportmonks_football_get_past_fixtures_by_tv_station","sportmonks_football_get_player","sportmonks_football_get_players_by_country","sportmonks_football_get_postmatch_news","sportmonks_football_get_postmatch_news_by_season","sportmonks_football_get_predictability_by_league","sportmonks_football_get_prematch_news","sportmonks_football_get_prematch_news_by_season","sportmonks_football_get_prematch_news_upcoming","sportmonks_football_get_probabilities","sportmonks_football_get_probabilities_by_fixture","sportmonks_football_get_referee","sportmonks_football_get_referees","sportmonks_football_get_referees_by_country","sportmonks_football_get_referees_by_season","sportmonks_football_get_rivals_by_team","sportmonks_football_get_round","sportmonks_football_get_round_statistics","sportmonks_football_get_rounds","sportmonks_football_get_rounds_by_season","sportmonks_football_get_schedules_by_season","sportmonks_football_get_schedules_by_season_and_team","sportmonks_football_get_schedules_by_team","sportmonks_football_get_season","sportmonks_football_get_seasons","sportmonks_football_get_seasons_by_team","sportmonks_football_get_stage","sportmonks_football_get_stage_statistics","sportmonks_football_get_stages","sportmonks_football_get_stages_by_season","sportmonks_football_get_standing_corrections_by_season","sportmonks_football_get_standings","sportmonks_football_get_standings_by_round","sportmonks_football_get_standings_by_season","sportmonks_football_get_state","sportmonks_football_get_states","sportmonks_football_get_team","sportmonks_football_get_team_rankings","sportmonks_football_get_team_rankings_by_date","sportmonks_football_get_team_rankings_by_team","sportmonks_football_get_team_squad","sportmonks_football_get_team_squad_by_season","sportmonks_football_get_teams_by_country","sportmonks_football_get_teams_by_season","sportmonks_football_get_topscorers_by_season","sportmonks_football_get_topscorers_by_stage","sportmonks_football_get_totw","sportmonks_football_get_totw_by_round","sportmonks_football_get_transfer","sportmonks_football_get_transfer_rumour","sportmonks_football_get_transfer_rumours_between_dates","sportmonks_football_get_transfer_rumours_by_player","sportmonks_football_get_transfer_rumours_by_team","sportmonks_football_get_transfers_between_dates","sportmonks_football_get_transfers_by_player","sportmonks_football_get_transfers_by_team","sportmonks_football_get_tv_station","sportmonks_football_get_tv_stations","sportmonks_football_get_tv_stations_by_fixture","sportmonks_football_get_upcoming_fixtures_by_market","sportmonks_football_get_upcoming_fixtures_by_tv_station","sportmonks_football_get_value_bets","sportmonks_football_get_value_bets_by_fixture","sportmonks_football_get_venue","sportmonks_football_get_venues","sportmonks_football_get_venues_by_season","sportmonks_football_search_coaches","sportmonks_football_search_fixtures","sportmonks_football_search_leagues","sportmonks_football_search_players","sportmonks_football_search_referees","sportmonks_football_search_rounds","sportmonks_football_search_seasons","sportmonks_football_search_stages","sportmonks_football_search_teams","sportmonks_football_search_venues","sportmonks_motorsport_get_all_fixtures","sportmonks_motorsport_get_current_leagues_by_team","sportmonks_motorsport_get_driver","sportmonks_motorsport_get_driver_standings","sportmonks_motorsport_get_driver_standings_by_season","sportmonks_motorsport_get_drivers","sportmonks_motorsport_get_drivers_by_country","sportmonks_motorsport_get_drivers_by_season","sportmonks_motorsport_get_fixture","sportmonks_motorsport_get_fixtures_by_date","sportmonks_motorsport_get_fixtures_by_date_range","sportmonks_motorsport_get_fixtures_by_ids","sportmonks_motorsport_get_laps_by_fixture","sportmonks_motorsport_get_laps_by_fixture_and_driver","sportmonks_motorsport_get_laps_by_fixture_and_lap","sportmonks_motorsport_get_latest_laps_by_fixture","sportmonks_motorsport_get_latest_pitstops_by_fixture","sportmonks_motorsport_get_latest_stints_by_fixture","sportmonks_motorsport_get_latest_updated_drivers","sportmonks_motorsport_get_latest_updated_fixtures","sportmonks_motorsport_get_league","sportmonks_motorsport_get_leagues","sportmonks_motorsport_get_leagues_by_country","sportmonks_motorsport_get_leagues_by_date","sportmonks_motorsport_get_leagues_by_live","sportmonks_motorsport_get_leagues_by_team","sportmonks_motorsport_get_livescores","sportmonks_motorsport_get_pitstops_by_fixture","sportmonks_motorsport_get_pitstops_by_fixture_and_driver","sportmonks_motorsport_get_pitstops_by_fixture_and_lap","sportmonks_motorsport_get_race_results_by_season_and_driver","sportmonks_motorsport_get_race_results_by_season_and_team","sportmonks_motorsport_get_schedules_by_season","sportmonks_motorsport_get_season","sportmonks_motorsport_get_seasons","sportmonks_motorsport_get_stage","sportmonks_motorsport_get_stages","sportmonks_motorsport_get_stages_by_season","sportmonks_motorsport_get_state","sportmonks_motorsport_get_states","sportmonks_motorsport_get_stints_by_fixture","sportmonks_motorsport_get_stints_by_fixture_and_driver","sportmonks_motorsport_get_stints_by_fixture_and_stint","sportmonks_motorsport_get_team","sportmonks_motorsport_get_team_standings","sportmonks_motorsport_get_team_standings_by_season","sportmonks_motorsport_get_teams","sportmonks_motorsport_get_teams_by_country","sportmonks_motorsport_get_teams_by_season","sportmonks_motorsport_get_venue","sportmonks_motorsport_get_venues","sportmonks_motorsport_get_venues_by_season","sportmonks_motorsport_search_drivers","sportmonks_motorsport_search_leagues","sportmonks_motorsport_search_stages","sportmonks_motorsport_search_teams","sportmonks_motorsport_search_venues","sportmonks_odds_get_all_historical_odds","sportmonks_odds_get_all_inplay_odds","sportmonks_odds_get_all_pre_match_odds","sportmonks_odds_get_all_premium_odds","sportmonks_odds_get_bookmaker","sportmonks_odds_get_bookmaker_event_ids_by_fixture","sportmonks_odds_get_bookmakers","sportmonks_odds_get_bookmakers_by_fixture","sportmonks_odds_get_inplay_odds_by_fixture","sportmonks_odds_get_inplay_odds_by_fixture_and_bookmaker","sportmonks_odds_get_inplay_odds_by_fixture_and_market","sportmonks_odds_get_last_updated_inplay_odds","sportmonks_odds_get_last_updated_pre_match_odds","sportmonks_odds_get_market","sportmonks_odds_get_markets","sportmonks_odds_get_pre_match_odds_by_fixture","sportmonks_odds_get_pre_match_odds_by_fixture_and_bookmaker","sportmonks_odds_get_pre_match_odds_by_fixture_and_market","sportmonks_odds_get_premium_odds_by_fixture","sportmonks_odds_get_premium_odds_by_fixture_and_bookmaker","sportmonks_odds_get_premium_odds_by_fixture_and_market","sportmonks_odds_get_updated_historical_odds_between","sportmonks_odds_get_updated_premium_odds_between","sportmonks_odds_search_bookmakers","sportmonks_odds_search_markets","spotify_add_playlist_cover","spotify_add_to_queue","spotify_add_tracks_to_playlist","spotify_check_following","spotify_check_playlist_followers","spotify_check_saved_albums","spotify_check_saved_audiobooks","spotify_check_saved_episodes","spotify_check_saved_shows","spotify_check_saved_tracks","spotify_create_playlist","spotify_follow_artists","spotify_follow_playlist","spotify_get_album","spotify_get_album_tracks","spotify_get_albums","spotify_get_artist","spotify_get_artist_albums","spotify_get_artist_top_tracks","spotify_get_artists","spotify_get_audiobook","spotify_get_audiobook_chapters","spotify_get_audiobooks","spotify_get_categories","spotify_get_current_user","spotify_get_currently_playing","spotify_get_devices","spotify_get_episode","spotify_get_episodes","spotify_get_followed_artists","spotify_get_markets","spotify_get_new_releases","spotify_get_playback_state","spotify_get_playlist","spotify_get_playlist_cover","spotify_get_playlist_tracks","spotify_get_queue","spotify_get_recently_played","spotify_get_saved_albums","spotify_get_saved_audiobooks","spotify_get_saved_episodes","spotify_get_saved_shows","spotify_get_saved_tracks","spotify_get_show","spotify_get_show_episodes","spotify_get_shows","spotify_get_top_artists","spotify_get_top_tracks","spotify_get_track","spotify_get_tracks","spotify_get_user_playlists","spotify_get_user_profile","spotify_pause","spotify_play","spotify_remove_saved_albums","spotify_remove_saved_audiobooks","spotify_remove_saved_episodes","spotify_remove_saved_shows","spotify_remove_saved_tracks","spotify_remove_tracks_from_playlist","spotify_reorder_playlist_items","spotify_replace_playlist_items","spotify_save_albums","spotify_save_audiobooks","spotify_save_episodes","spotify_save_shows","spotify_save_tracks","spotify_search","spotify_seek","spotify_set_repeat","spotify_set_shuffle","spotify_set_volume","spotify_skip_next","spotify_skip_previous","spotify_transfer_playback","spotify_unfollow_artists","spotify_unfollow_playlist","spotify_update_playlist","sqs_send","square_batch_retrieve_inventory_counts","square_cancel_invoice","square_cancel_payment","square_complete_payment","square_create_catalog_image","square_create_customer","square_create_invoice","square_create_order","square_create_payment","square_delete_catalog_object","square_delete_customer","square_delete_invoice","square_get_catalog_object","square_get_customer","square_get_invoice","square_get_location","square_get_order","square_get_payment","square_get_refund","square_list_catalog","square_list_customers","square_list_invoices","square_list_locations","square_list_payments","square_list_refunds","square_pay_order","square_publish_invoice","square_refund_payment","square_search_catalog_objects","square_search_customers","square_search_invoices","square_search_orders","square_update_customer","square_upsert_catalog_object","ssh_check_command_exists","ssh_check_file_exists","ssh_create_directory","ssh_delete_file","ssh_download_file","ssh_execute_command","ssh_execute_script","ssh_get_system_info","ssh_list_directory","ssh_move_rename","ssh_read_file_content","ssh_upload_file","ssh_write_file_content","stagehand_agent","stagehand_extract","stripe_cancel_payment_intent","stripe_cancel_subscription","stripe_capture_charge","stripe_capture_payment_intent","stripe_confirm_payment_intent","stripe_create_charge","stripe_create_customer","stripe_create_invoice","stripe_create_payment_intent","stripe_create_price","stripe_create_product","stripe_create_subscription","stripe_delete_customer","stripe_delete_invoice","stripe_delete_product","stripe_finalize_invoice","stripe_list_charges","stripe_list_customers","stripe_list_events","stripe_list_invoices","stripe_list_payment_intents","stripe_list_prices","stripe_list_products","stripe_list_subscriptions","stripe_pay_invoice","stripe_resume_subscription","stripe_retrieve_charge","stripe_retrieve_customer","stripe_retrieve_event","stripe_retrieve_invoice","stripe_retrieve_payment_intent","stripe_retrieve_price","stripe_retrieve_product","stripe_retrieve_subscription","stripe_search_charges","stripe_search_customers","stripe_search_invoices","stripe_search_payment_intents","stripe_search_prices","stripe_search_products","stripe_search_subscriptions","stripe_send_invoice","stripe_update_charge","stripe_update_customer","stripe_update_invoice","stripe_update_payment_intent","stripe_update_price","stripe_update_product","stripe_update_subscription","stripe_void_invoice","sts_assume_role","sts_assume_role_with_saml","sts_assume_role_with_web_identity","sts_get_access_key_info","sts_get_caller_identity","sts_get_session_token","stt_assemblyai","stt_assemblyai_v2","stt_deepgram","stt_deepgram_v2","stt_elevenlabs","stt_elevenlabs_v2","stt_gemini","stt_gemini_v2","stt_whisper","stt_whisper_v2","supabase_count","supabase_delete","supabase_get_row","supabase_insert","supabase_introspect","supabase_invoke_function","supabase_query","supabase_rpc","supabase_storage_copy","supabase_storage_create_bucket","supabase_storage_create_signed_upload_url","supabase_storage_create_signed_url","supabase_storage_delete","supabase_storage_delete_bucket","supabase_storage_download","supabase_storage_empty_bucket","supabase_storage_get_public_url","supabase_storage_list","supabase_storage_list_buckets","supabase_storage_move","supabase_storage_update_bucket","supabase_storage_upload","supabase_text_search","supabase_update","supabase_upsert","supabase_vector_search","table_batch_insert_rows","table_create","table_delete_row","table_delete_rows_by_filter","table_get_row","table_get_schema","table_insert_row","table_list","table_query_rows","table_query_rows_v2","table_update_row","table_update_rows_by_filter","table_upsert_row","tailscale_authorize_device","tailscale_create_auth_key","tailscale_delete_auth_key","tailscale_delete_device","tailscale_delete_user","tailscale_expire_device_key","tailscale_get_acl","tailscale_get_auth_key","tailscale_get_device","tailscale_get_device_routes","tailscale_get_dns_preferences","tailscale_get_dns_searchpaths","tailscale_list_auth_keys","tailscale_list_devices","tailscale_list_dns_nameservers","tailscale_list_users","tailscale_set_acl","tailscale_set_device_routes","tailscale_set_device_tags","tailscale_set_dns_nameservers","tailscale_set_dns_preferences","tailscale_set_dns_searchpaths","tailscale_suspend_user","tailscale_update_device_key","tavily_crawl","tavily_extract","tavily_map","tavily_search","telegram_copy_message","telegram_delete_message","telegram_edit_message_text","telegram_forward_message","telegram_get_chat","telegram_get_chat_member","telegram_message","telegram_pin_message","telegram_send_animation","telegram_send_audio","telegram_send_chat_action","telegram_send_contact","telegram_send_document","telegram_send_location","telegram_send_photo","telegram_send_poll","telegram_send_video","telegram_set_message_reaction","telegram_unpin_message","temporal_cancel_workflow","temporal_count_workflows","temporal_create_schedule","temporal_delete_schedule","temporal_describe_schedule","temporal_describe_task_queue","temporal_describe_workflow","temporal_get_workflow_history","temporal_list_schedules","temporal_list_workflows","temporal_pause_schedule","temporal_query_workflow","temporal_reset_workflow","temporal_signal_with_start","temporal_signal_workflow","temporal_start_workflow","temporal_terminate_workflow","temporal_trigger_schedule","temporal_unpause_schedule","temporal_update_workflow","textract_analyze_expense","textract_analyze_id","textract_parser","textract_parser_v2","thinking_tool","thrive_add_audience_managers","thrive_add_audience_members","thrive_add_user_tags","thrive_create_assignment","thrive_create_audience","thrive_create_completion","thrive_create_user","thrive_delete_assignment","thrive_delete_audience","thrive_delete_user","thrive_get_activity","thrive_get_assignment","thrive_get_audience","thrive_get_completion","thrive_get_content","thrive_get_cpd_category","thrive_get_cpd_entry","thrive_get_cpd_requirement","thrive_get_enrolment","thrive_get_skill_levels","thrive_get_tag","thrive_get_user_by_id","thrive_get_user_by_ref","thrive_list_assignments","thrive_list_audience_managers","thrive_list_audience_members","thrive_list_audiences","thrive_list_completions","thrive_list_enrolments","thrive_list_tags","thrive_query_activities","thrive_query_content","thrive_query_cpd_categories","thrive_query_cpd_entries","thrive_query_cpd_requirements","thrive_query_cpd_user_summaries","thrive_remove_audience_manager","thrive_remove_audience_member","thrive_remove_user_tags","thrive_replace_audience_managers","thrive_replace_audience_members","thrive_search_users","thrive_suspend_user","thrive_update_assignment","thrive_update_audience","thrive_update_user","thrive_update_user_skills","tiktok_get_post_status","tiktok_get_user","tiktok_list_videos","tiktok_query_videos","tiktok_upload_video_draft","tinybird_append_datasource","tinybird_delete_datasource_rows","tinybird_events","tinybird_get_job","tinybird_query","tinybird_query_pipe","tinybird_truncate_datasource","trello_add_checklist","trello_add_checklist_item","trello_add_comment","trello_add_label","trello_add_member","trello_create_board","trello_create_card","trello_create_list","trello_delete_card","trello_get_actions","trello_get_board","trello_get_card","trello_list_cards","trello_list_lists","trello_list_members","trello_remove_label","trello_remove_member","trello_search","trello_update_card","trello_update_checklist_item","trello_update_list","trigger_dev_activate_schedule","trigger_dev_add_run_tags","trigger_dev_batch_trigger_task","trigger_dev_cancel_run","trigger_dev_complete_waitpoint_token","trigger_dev_create_env_var","trigger_dev_create_schedule","trigger_dev_create_waitpoint_token","trigger_dev_deactivate_schedule","trigger_dev_delete_env_var","trigger_dev_delete_schedule","trigger_dev_execute_query","trigger_dev_get_batch","trigger_dev_get_batch_results","trigger_dev_get_deployment","trigger_dev_get_env_var","trigger_dev_get_latest_deployment","trigger_dev_get_query_schema","trigger_dev_get_queue","trigger_dev_get_run","trigger_dev_get_run_events","trigger_dev_get_run_result","trigger_dev_get_run_trace","trigger_dev_get_schedule","trigger_dev_get_waitpoint_token","trigger_dev_import_env_vars","trigger_dev_list_deployments","trigger_dev_list_env_vars","trigger_dev_list_queues","trigger_dev_list_runs","trigger_dev_list_schedules","trigger_dev_list_timezones","trigger_dev_list_waitpoint_tokens","trigger_dev_override_queue_concurrency","trigger_dev_pause_queue","trigger_dev_promote_deployment","trigger_dev_replay_run","trigger_dev_reschedule_run","trigger_dev_reset_queue_concurrency","trigger_dev_resume_queue","trigger_dev_trigger_task","trigger_dev_update_env_var","trigger_dev_update_run_metadata","trigger_dev_update_schedule","tts_azure","tts_cartesia","tts_deepgram","tts_elevenlabs","tts_google","tts_openai","tts_playht","twilio_send_sms","twilio_voice_get_recording","twilio_voice_list_calls","twilio_voice_make_call","typeform_create_form","typeform_delete_form","typeform_files","typeform_get_form","typeform_insights","typeform_list_forms","typeform_responses","typeform_update_form","upstash_redis_command","upstash_redis_delete","upstash_redis_exists","upstash_redis_expire","upstash_redis_get","upstash_redis_hget","upstash_redis_hgetall","upstash_redis_hset","upstash_redis_incr","upstash_redis_incrby","upstash_redis_keys","upstash_redis_lpush","upstash_redis_lrange","upstash_redis_set","upstash_redis_setnx","upstash_redis_ttl","uptimerobot_create_alert_contact","uptimerobot_create_maintenance_window","uptimerobot_create_monitor","uptimerobot_create_psp","uptimerobot_delete_alert_contact","uptimerobot_delete_maintenance_window","uptimerobot_delete_monitor","uptimerobot_delete_psp","uptimerobot_get_account","uptimerobot_get_alert_contact","uptimerobot_get_incident","uptimerobot_get_maintenance_window","uptimerobot_get_monitor","uptimerobot_get_psp","uptimerobot_list_alert_contacts","uptimerobot_list_incidents","uptimerobot_list_maintenance_windows","uptimerobot_list_monitors","uptimerobot_list_psps","uptimerobot_pause_monitor","uptimerobot_start_monitor","uptimerobot_update_maintenance_window","uptimerobot_update_monitor","uptimerobot_update_psp","vanta_download_document_file","vanta_get_control","vanta_get_document","vanta_get_framework","vanta_get_person","vanta_get_policy","vanta_get_risk_scenario","vanta_get_test","vanta_get_vendor","vanta_get_vulnerable_asset","vanta_list_control_documents","vanta_list_control_tests","vanta_list_controls","vanta_list_document_uploads","vanta_list_documents","vanta_list_framework_controls","vanta_list_frameworks","vanta_list_monitored_computers","vanta_list_people","vanta_list_policies","vanta_list_risk_scenarios","vanta_list_test_entities","vanta_list_tests","vanta_list_vendors","vanta_list_vulnerabilities","vanta_list_vulnerability_remediations","vanta_list_vulnerable_assets","vanta_submit_document","vanta_upload_document_file","vercel_add_domain","vercel_add_project_domain","vercel_cancel_deployment","vercel_create_alias","vercel_create_check","vercel_create_deployment","vercel_create_dns_record","vercel_create_edge_config","vercel_create_env_var","vercel_create_project","vercel_create_webhook","vercel_delete_alias","vercel_delete_deployment","vercel_delete_dns_record","vercel_delete_domain","vercel_delete_edge_config","vercel_delete_env_var","vercel_delete_project","vercel_delete_webhook","vercel_get_alias","vercel_get_check","vercel_get_deployment","vercel_get_deployment_events","vercel_get_domain","vercel_get_domain_config","vercel_get_edge_config","vercel_get_edge_config_items","vercel_get_env_vars","vercel_get_project","vercel_get_team","vercel_get_user","vercel_get_webhook","vercel_list_aliases","vercel_list_checks","vercel_list_deployment_files","vercel_list_deployments","vercel_list_dns_records","vercel_list_domains","vercel_list_edge_configs","vercel_list_project_domains","vercel_list_projects","vercel_list_team_members","vercel_list_teams","vercel_list_webhooks","vercel_pause_project","vercel_promote_deployment","vercel_remove_project_domain","vercel_rerequest_check","vercel_unpause_project","vercel_update_check","vercel_update_dns_record","vercel_update_edge_config_items","vercel_update_env_var","vercel_update_project","vercel_update_project_domain","vercel_verify_project_domain","video_falai","video_luma","video_minimax","video_runway","video_veo","vision_tool","vision_tool_v2","wealthbox_read_contact","wealthbox_read_note","wealthbox_read_task","wealthbox_write_contact","wealthbox_write_note","wealthbox_write_task","webflow_create_item","webflow_delete_item","webflow_get_item","webflow_list_items","webflow_update_item","webhook_request","whatsapp_get_media","whatsapp_mark_read","whatsapp_send_interactive","whatsapp_send_media","whatsapp_send_message","whatsapp_send_reaction","whatsapp_send_template","whatsapp_upload_media","wikipedia_content","wikipedia_random","wikipedia_search","wikipedia_summary","windchill_check_in_document","windchill_check_in_documents","windchill_check_out_document","windchill_check_out_documents","windchill_create_document","windchill_create_documents","windchill_delete_document","windchill_delete_documents","windchill_download_attachment","windchill_download_primary_content","windchill_get_document","windchill_get_document_structure","windchill_get_primary_content","windchill_get_valid_state_transitions","windchill_list_attachments","windchill_list_documents","windchill_revise_document","windchill_revise_documents","windchill_set_lifecycle_state","windchill_undo_check_out_document","windchill_undo_check_out_documents","windchill_update_common_properties","windchill_update_document","windchill_update_document_security_labels","windchill_update_documents","windchill_upload_attachments","windchill_upload_primary_content","wiza_company_enrichment","wiza_get_credits","wiza_individual_reveal","wiza_prospect_search","wordpress_create_category","wordpress_create_comment","wordpress_create_page","wordpress_create_post","wordpress_create_tag","wordpress_delete_category","wordpress_delete_comment","wordpress_delete_media","wordpress_delete_page","wordpress_delete_post","wordpress_delete_tag","wordpress_get_category","wordpress_get_current_user","wordpress_get_media","wordpress_get_page","wordpress_get_post","wordpress_get_tag","wordpress_get_user","wordpress_list_categories","wordpress_list_comments","wordpress_list_media","wordpress_list_pages","wordpress_list_posts","wordpress_list_tags","wordpress_list_users","wordpress_search_content","wordpress_update_category","wordpress_update_comment","wordpress_update_page","wordpress_update_post","wordpress_update_tag","wordpress_upload_media","workday_assign_onboarding","workday_change_job","workday_create_prehire","workday_get_compensation","workday_get_organizations","workday_get_worker","workday_hire_employee","workday_list_workers","workday_terminate_worker","workday_update_worker","workflow_executor","x_create_bookmark","x_create_tweet","x_delete_bookmark","x_delete_tweet","x_get_blocking","x_get_bookmarks","x_get_followers","x_get_following","x_get_liked_tweets","x_get_liking_users","x_get_me","x_get_personalized_trends","x_get_quote_tweets","x_get_retweeted_by","x_get_trends_by_woeid","x_get_tweets_by_ids","x_get_usage","x_get_user_mentions","x_get_user_timeline","x_get_user_tweets","x_hide_reply","x_manage_block","x_manage_follow","x_manage_like","x_manage_mute","x_manage_retweet","x_read","x_search","x_search_tweets","x_search_users","x_user","x_write","youtube_channel_info","youtube_channel_playlists","youtube_channel_videos","youtube_comments","youtube_playlist_items","youtube_search","youtube_trending","youtube_video_categories","youtube_video_details","zendesk_autocomplete_organizations","zendesk_create_organization","zendesk_create_organizations_bulk","zendesk_create_ticket","zendesk_create_tickets_bulk","zendesk_create_user","zendesk_create_users_bulk","zendesk_delete_organization","zendesk_delete_ticket","zendesk_delete_user","zendesk_get_current_user","zendesk_get_organization","zendesk_get_organizations","zendesk_get_ticket","zendesk_get_tickets","zendesk_get_user","zendesk_get_users","zendesk_merge_tickets","zendesk_search","zendesk_search_count","zendesk_search_users","zendesk_update_organization","zendesk_update_ticket","zendesk_update_tickets_bulk","zendesk_update_user","zendesk_update_users_bulk","zep_add_messages","zep_add_user","zep_create_thread","zep_delete_thread","zep_get_context","zep_get_messages","zep_get_threads","zep_get_user","zep_get_user_threads","zerobounce_get_credits","zerobounce_verify_email","zoho_desk_add_comment","zoho_desk_get_attachment","zoho_desk_get_contact","zoho_desk_get_thread","zoho_desk_get_ticket","zoho_desk_list_comments","zoho_desk_list_organizations","zoho_desk_list_threads","zoho_desk_list_tickets","zoho_desk_update_ticket","zoom_create_meeting","zoom_delete_meeting","zoom_delete_recording","zoom_get_meeting","zoom_get_meeting_invitation","zoom_get_meeting_recordings","zoom_list_meetings","zoom_list_past_participants","zoom_list_recordings","zoom_update_meeting","zoominfo_enrich_companies","zoominfo_enrich_contacts","zoominfo_search_companies","zoominfo_search_contacts","zoominfo_search_intent","zoominfo_search_news"]' + '["a2a_cancel_task","a2a_get_agent_card","a2a_get_task","a2a_send_message","agentmail_create_draft","agentmail_create_inbox","agentmail_delete_draft","agentmail_delete_inbox","agentmail_delete_thread","agentmail_forward_message","agentmail_get_draft","agentmail_get_inbox","agentmail_get_message","agentmail_get_thread","agentmail_list_drafts","agentmail_list_inboxes","agentmail_list_messages","agentmail_list_threads","agentmail_reply_message","agentmail_send_draft","agentmail_send_message","agentmail_update_draft","agentmail_update_inbox","agentmail_update_message","agentmail_update_thread","agentphone_create_call","agentphone_create_contact","agentphone_create_number","agentphone_delete_contact","agentphone_get_call","agentphone_get_call_transcript","agentphone_get_contact","agentphone_get_conversation","agentphone_get_conversation_messages","agentphone_get_number_messages","agentphone_get_usage","agentphone_get_usage_daily","agentphone_get_usage_monthly","agentphone_list_calls","agentphone_list_contacts","agentphone_list_conversations","agentphone_list_numbers","agentphone_react_to_message","agentphone_release_number","agentphone_send_message","agentphone_update_contact","agentphone_update_conversation","agiloft_async_status","agiloft_attach_file","agiloft_attachment_info","agiloft_create_record","agiloft_delete_record","agiloft_get_choice_line_id","agiloft_list_tables","agiloft_lock_record","agiloft_nlp_search","agiloft_read_record","agiloft_remove_attachment","agiloft_retrieve_attachment","agiloft_run_action_button","agiloft_saved_search","agiloft_search_records","agiloft_select_records","agiloft_update_record","agiloft_upsert_record","ahrefs_anchors","ahrefs_backlinks","ahrefs_backlinks_stats","ahrefs_batch_analysis","ahrefs_broken_backlinks","ahrefs_domain_rating","ahrefs_domain_rating_history","ahrefs_keyword_overview","ahrefs_keywords_history","ahrefs_metrics","ahrefs_metrics_history","ahrefs_organic_competitors","ahrefs_organic_keywords","ahrefs_paid_pages","ahrefs_rank_tracker_competitors_overview","ahrefs_rank_tracker_competitors_stats","ahrefs_rank_tracker_overview","ahrefs_rank_tracker_serp_overview","ahrefs_refdomains_history","ahrefs_referring_domains","ahrefs_related_terms","ahrefs_site_audit_page_explorer","ahrefs_top_pages","airtable_create_records","airtable_delete_records","airtable_get_base_schema","airtable_get_record","airtable_list_bases","airtable_list_records","airtable_list_tables","airtable_update_multiple_records","airtable_update_record","airtable_upsert_records","airweave_search","algolia_add_record","algolia_batch_operations","algolia_browse_records","algolia_clear_records","algolia_copy_move_index","algolia_delete_by_filter","algolia_delete_index","algolia_delete_record","algolia_get_record","algolia_get_records","algolia_get_settings","algolia_get_task_status","algolia_list_indices","algolia_partial_update_record","algolia_search","algolia_update_settings","amplitude_event_segmentation","amplitude_funnels","amplitude_get_active_users","amplitude_get_revenue","amplitude_group_identify","amplitude_identify_user","amplitude_list_events","amplitude_realtime_active_users","amplitude_retention","amplitude_send_event","amplitude_user_activity","amplitude_user_profile","amplitude_user_search","apify_get_dataset_items","apify_get_run","apify_run_actor_async","apify_run_actor_sync","apify_run_task","apollo_account_bulk_create","apollo_account_bulk_update","apollo_account_create","apollo_account_search","apollo_account_update","apollo_contact_bulk_create","apollo_contact_bulk_update","apollo_contact_create","apollo_contact_search","apollo_contact_update","apollo_email_accounts","apollo_opportunity_create","apollo_opportunity_get","apollo_opportunity_search","apollo_opportunity_update","apollo_organization_bulk_enrich","apollo_organization_enrich","apollo_organization_search","apollo_people_bulk_enrich","apollo_people_enrich","apollo_people_search","apollo_sequence_add_contacts","apollo_sequence_search","apollo_task_create","apollo_task_search","appconfig_create_application","appconfig_create_configuration_profile","appconfig_create_environment","appconfig_create_hosted_configuration_version","appconfig_delete_application","appconfig_delete_configuration_profile","appconfig_delete_environment","appconfig_delete_hosted_configuration_version","appconfig_get_application","appconfig_get_configuration","appconfig_get_configuration_profile","appconfig_get_deployment","appconfig_get_environment","appconfig_get_hosted_configuration_version","appconfig_list_applications","appconfig_list_configuration_profiles","appconfig_list_deployment_strategies","appconfig_list_deployments","appconfig_list_environments","appconfig_list_hosted_configuration_versions","appconfig_start_deployment","appconfig_stop_deployment","appconfig_update_application","appconfig_update_configuration_profile","appconfig_update_environment","arxiv_get_author_papers","arxiv_get_paper","arxiv_search","asana_add_comment","asana_add_followers","asana_create_project","asana_create_section","asana_create_subtask","asana_create_task","asana_delete_task","asana_get_project","asana_get_projects","asana_get_task","asana_list_sections","asana_list_workspaces","asana_search_tasks","asana_update_task","ashby_add_candidate_tag","ashby_anonymize_candidate","ashby_change_application_source","ashby_change_application_stage","ashby_create_application","ashby_create_candidate","ashby_create_note","ashby_delete_application","ashby_get_application","ashby_get_candidate","ashby_get_job","ashby_get_job_posting","ashby_get_offer","ashby_list_applications","ashby_list_archive_reasons","ashby_list_candidate_tags","ashby_list_candidates","ashby_list_custom_fields","ashby_list_departments","ashby_list_interviews","ashby_list_job_postings","ashby_list_jobs","ashby_list_locations","ashby_list_notes","ashby_list_offers","ashby_list_openings","ashby_list_sources","ashby_list_users","ashby_remove_candidate_tag","ashby_search_candidates","ashby_set_custom_field_value","ashby_set_custom_field_values","ashby_update_candidate","athena_batch_get_query_execution","athena_create_named_query","athena_delete_named_query","athena_get_named_query","athena_get_query_execution","athena_get_query_results","athena_list_databases","athena_list_named_queries","athena_list_query_executions","athena_list_table_metadata","athena_start_query","athena_stop_query","attio_assert_record","attio_create_attribute","attio_create_comment","attio_create_list","attio_create_list_entry","attio_create_note","attio_create_object","attio_create_record","attio_create_task","attio_create_webhook","attio_delete_comment","attio_delete_list_entry","attio_delete_note","attio_delete_record","attio_delete_task","attio_delete_webhook","attio_get_attribute","attio_get_comment","attio_get_list","attio_get_list_entry","attio_get_member","attio_get_note","attio_get_object","attio_get_record","attio_get_task","attio_get_thread","attio_get_webhook","attio_list_attributes","attio_list_lists","attio_list_members","attio_list_notes","attio_list_objects","attio_list_records","attio_list_tasks","attio_list_threads","attio_list_webhooks","attio_query_list_entries","attio_search_records","attio_update_attribute","attio_update_list","attio_update_list_entry","attio_update_object","attio_update_record","attio_update_task","attio_update_webhook","azure_data_explorer_create_table","azure_data_explorer_drop_table","azure_data_explorer_ingest_from_query","azure_data_explorer_ingest_inline","azure_data_explorer_list_databases","azure_data_explorer_list_functions","azure_data_explorer_list_tables","azure_data_explorer_management","azure_data_explorer_query","azure_data_explorer_show_database_schema","azure_data_explorer_show_ingestion_failures","azure_data_explorer_show_operations","azure_data_explorer_show_table_details","azure_data_explorer_show_table_schema","azure_devops_add_comment","azure_devops_create_work_item","azure_devops_get_build_log","azure_devops_get_build_timeline","azure_devops_get_comments","azure_devops_get_pipeline","azure_devops_get_pipeline_run","azure_devops_get_work_item","azure_devops_get_work_items_batch","azure_devops_get_work_items_between_builds","azure_devops_list_build_logs","azure_devops_list_builds","azure_devops_list_pipeline_runs","azure_devops_list_pipelines","azure_devops_query_work_items","azure_devops_update_work_item","box_copy_file","box_create_folder","box_delete_file","box_delete_folder","box_download_file","box_get_file_info","box_list_folder_items","box_search","box_sign_cancel_request","box_sign_create_request","box_sign_get_request","box_sign_list_requests","box_sign_resend_request","box_update_file","box_upload_file","brandfetch_get_brand","brandfetch_search","brex_archive_budget","brex_create_budget","brex_create_spend_limit","brex_create_transfer","brex_create_vendor","brex_get_budget","brex_get_cash_account","brex_get_company","brex_get_current_user","brex_get_expense","brex_get_spend_limit","brex_get_transfer","brex_get_user","brex_get_vendor","brex_list_budgets","brex_list_card_accounts","brex_list_card_statements","brex_list_card_transactions","brex_list_cards","brex_list_cash_accounts","brex_list_cash_statements","brex_list_cash_transactions","brex_list_departments","brex_list_expenses","brex_list_locations","brex_list_spend_limits","brex_list_titles","brex_list_transfers","brex_list_users","brex_list_vendors","brex_match_receipt","brex_update_expense","brex_update_vendor","brex_upload_receipt","brightdata_cancel_snapshot","brightdata_discover","brightdata_download_snapshot","brightdata_scrape_dataset","brightdata_scrape_url","brightdata_serp_search","brightdata_snapshot_status","brightdata_sync_scrape","browser_use_run_task","buffer_create_idea","buffer_create_post","buffer_delete_post","buffer_edit_post","buffer_get_account","buffer_get_channels","buffer_get_idea_groups","buffer_get_ideas","buffer_get_post","buffer_get_posts","calcom_cancel_booking","calcom_confirm_booking","calcom_create_booking","calcom_create_event_type","calcom_create_schedule","calcom_decline_booking","calcom_delete_event_type","calcom_delete_schedule","calcom_get_booking","calcom_get_default_schedule","calcom_get_event_type","calcom_get_schedule","calcom_get_slots","calcom_list_bookings","calcom_list_event_types","calcom_list_schedules","calcom_reschedule_booking","calcom_update_event_type","calcom_update_schedule","calendly_cancel_event","calendly_create_event_invitee","calendly_create_invitee_no_show","calendly_create_scheduling_link","calendly_create_webhook","calendly_delete_invitee_no_show","calendly_delete_webhook","calendly_get_current_user","calendly_get_event_invitee","calendly_get_event_type","calendly_get_scheduled_event","calendly_get_user","calendly_list_event_invitees","calendly_list_event_type_available_times","calendly_list_event_types","calendly_list_organization_memberships","calendly_list_routing_form_submissions","calendly_list_routing_forms","calendly_list_scheduled_events","calendly_list_user_availability_schedules","calendly_list_user_busy_times","calendly_list_webhooks","clay_populate","clerk_add_organization_member","clerk_ban_user","clerk_create_actor_token","clerk_create_allowlist_identifier","clerk_create_blocklist_identifier","clerk_create_organization","clerk_create_organization_invitation","clerk_create_user","clerk_delete_allowlist_identifier","clerk_delete_blocklist_identifier","clerk_delete_organization","clerk_delete_user","clerk_get_jwt_template","clerk_get_organization","clerk_get_session","clerk_get_user","clerk_get_user_oauth_token","clerk_list_allowlist_identifiers","clerk_list_blocklist_identifiers","clerk_list_jwt_templates","clerk_list_organization_invitations","clerk_list_organization_memberships","clerk_list_organizations","clerk_list_sessions","clerk_list_users","clerk_lock_user","clerk_remove_organization_member","clerk_revoke_actor_token","clerk_revoke_session","clerk_unban_user","clerk_unlock_user","clerk_update_organization","clerk_update_organization_membership","clerk_update_user","clickhouse_count_rows","clickhouse_create_database","clickhouse_create_table","clickhouse_delete","clickhouse_describe_table","clickhouse_drop_database","clickhouse_drop_partition","clickhouse_drop_table","clickhouse_execute","clickhouse_insert","clickhouse_insert_rows","clickhouse_introspect","clickhouse_kill_query","clickhouse_list_clusters","clickhouse_list_databases","clickhouse_list_mutations","clickhouse_list_partitions","clickhouse_list_running_queries","clickhouse_list_tables","clickhouse_optimize_table","clickhouse_query","clickhouse_rename_table","clickhouse_show_create_table","clickhouse_table_stats","clickhouse_truncate_table","clickhouse_update","clickup_add_tag_to_task","clickup_create_checklist","clickup_create_checklist_item","clickup_create_comment","clickup_create_folder","clickup_create_list","clickup_create_task","clickup_create_time_entry","clickup_delete_checklist","clickup_delete_checklist_item","clickup_delete_comment","clickup_delete_task","clickup_delete_time_entry","clickup_get_comments","clickup_get_custom_fields","clickup_get_folders","clickup_get_list_members","clickup_get_lists","clickup_get_running_timer","clickup_get_space_tags","clickup_get_spaces","clickup_get_task","clickup_get_task_members","clickup_get_tasks","clickup_get_time_entries","clickup_get_workspaces","clickup_remove_custom_field_value","clickup_remove_tag_from_task","clickup_search_tasks","clickup_set_custom_field_value","clickup_start_timer","clickup_stop_timer","clickup_update_checklist","clickup_update_checklist_item","clickup_update_comment","clickup_update_task","clickup_update_time_entry","clickup_upload_attachment","cloudflare_create_access_application","cloudflare_create_access_policy","cloudflare_create_access_service_token","cloudflare_create_dns_record","cloudflare_create_r2_bucket","cloudflare_create_rate_limit_rule","cloudflare_create_ruleset","cloudflare_create_ruleset_rule","cloudflare_create_zone","cloudflare_delete_access_application","cloudflare_delete_access_policy","cloudflare_delete_dns_record","cloudflare_delete_r2_bucket","cloudflare_delete_ruleset_rule","cloudflare_delete_zone","cloudflare_dns_analytics","cloudflare_get_access_application","cloudflare_get_r2_bucket","cloudflare_get_ruleset","cloudflare_get_ruleset_entrypoint","cloudflare_get_tunnel","cloudflare_get_tunnel_configuration","cloudflare_get_worker_script_settings","cloudflare_get_zone","cloudflare_get_zone_settings","cloudflare_list_access_applications","cloudflare_list_access_groups","cloudflare_list_access_identity_providers","cloudflare_list_access_policies","cloudflare_list_access_service_tokens","cloudflare_list_certificates","cloudflare_list_dns_records","cloudflare_list_managed_ruleset_overrides","cloudflare_list_r2_buckets","cloudflare_list_rate_limit_rules","cloudflare_list_rulesets","cloudflare_list_tunnels","cloudflare_list_worker_routes","cloudflare_list_worker_scripts","cloudflare_list_zones","cloudflare_purge_cache","cloudflare_revoke_access_service_token","cloudflare_update_access_application","cloudflare_update_access_policy","cloudflare_update_dns_record","cloudflare_update_rate_limit_rule","cloudflare_update_ruleset_rule","cloudflare_update_zone_setting","cloudformation_cancel_update_stack","cloudformation_create_change_set","cloudformation_create_stack","cloudformation_delete_stack","cloudformation_describe_change_set","cloudformation_describe_stack_drift_detection_status","cloudformation_describe_stack_events","cloudformation_describe_stacks","cloudformation_detect_stack_drift","cloudformation_execute_change_set","cloudformation_get_template","cloudformation_get_template_summary","cloudformation_list_stack_resources","cloudformation_update_stack","cloudformation_validate_template","cloudwatch_describe_alarm_history","cloudwatch_describe_alarms","cloudwatch_describe_log_groups","cloudwatch_describe_log_streams","cloudwatch_filter_log_events","cloudwatch_get_log_events","cloudwatch_get_metric_statistics","cloudwatch_list_metrics","cloudwatch_mute_alarm","cloudwatch_put_log_group_retention","cloudwatch_put_metric_data","cloudwatch_query_logs","cloudwatch_unmute_alarm","codepipeline_disable_stage_transition","codepipeline_enable_stage_transition","codepipeline_get_pipeline","codepipeline_get_pipeline_execution","codepipeline_get_pipeline_state","codepipeline_list_action_executions","codepipeline_list_pipeline_executions","codepipeline_list_pipelines","codepipeline_put_approval_result","codepipeline_retry_stage_execution","codepipeline_start_execution","codepipeline_stop_execution","confluence_add_label","confluence_create_blogpost","confluence_create_comment","confluence_create_page","confluence_create_page_property","confluence_create_space","confluence_create_space_property","confluence_delete_attachment","confluence_delete_blogpost","confluence_delete_comment","confluence_delete_label","confluence_delete_page","confluence_delete_page_property","confluence_delete_space","confluence_delete_space_property","confluence_get_blogpost","confluence_get_page_ancestors","confluence_get_page_children","confluence_get_page_descendants","confluence_get_page_version","confluence_get_pages_by_label","confluence_get_space","confluence_get_task","confluence_get_user","confluence_list_attachments","confluence_list_blogposts","confluence_list_blogposts_in_space","confluence_list_comments","confluence_list_labels","confluence_list_page_properties","confluence_list_page_versions","confluence_list_pages_in_space","confluence_list_space_labels","confluence_list_space_permissions","confluence_list_space_properties","confluence_list_spaces","confluence_list_tasks","confluence_retrieve","confluence_search","confluence_search_in_space","confluence_update","confluence_update_blogpost","confluence_update_comment","confluence_update_space","confluence_update_task","confluence_upload_attachment","context_dev_classify_naics","context_dev_classify_sic","context_dev_crawl","context_dev_extract","context_dev_extract_product","context_dev_extract_products","context_dev_get_brand","context_dev_get_brand_by_email","context_dev_get_brand_by_name","context_dev_get_brand_by_ticker","context_dev_identify_transaction","context_dev_map","context_dev_scrape_fonts","context_dev_scrape_html","context_dev_scrape_images","context_dev_scrape_markdown","context_dev_scrape_styleguide","context_dev_screenshot","context_dev_search","convex_action","convex_document_deltas","convex_list_documents","convex_list_tables","convex_mutation","convex_query","convex_run_function","crowdstrike_create_indicators","crowdstrike_delete_indicators","crowdstrike_delete_rtr_session","crowdstrike_execute_rtr_command","crowdstrike_get_alert_details","crowdstrike_get_case_details","crowdstrike_get_host_group_details","crowdstrike_get_indicator_details","crowdstrike_get_rtr_command_status","crowdstrike_get_sensor_aggregates","crowdstrike_get_sensor_details","crowdstrike_get_vulnerability_details","crowdstrike_init_rtr_session","crowdstrike_perform_host_action","crowdstrike_perform_host_group_action","crowdstrike_query_alerts","crowdstrike_query_cases","crowdstrike_query_host_groups","crowdstrike_query_indicators","crowdstrike_query_sensors","crowdstrike_query_vulnerabilities","crowdstrike_update_alerts","crowdstrike_update_indicators","cursor_add_followup","cursor_add_followup_v2","cursor_delete_agent","cursor_delete_agent_v2","cursor_download_artifact","cursor_download_artifact_v2","cursor_get_agent","cursor_get_agent_v2","cursor_get_api_key_info","cursor_get_api_key_info_v2","cursor_get_conversation","cursor_get_conversation_v2","cursor_launch_agent","cursor_launch_agent_v2","cursor_list_agents","cursor_list_agents_v2","cursor_list_artifacts","cursor_list_artifacts_v2","cursor_list_models","cursor_list_models_v2","cursor_list_repositories","cursor_list_repositories_v2","cursor_stop_agent","cursor_stop_agent_v2","dagster_delete_run","dagster_get_asset","dagster_get_run","dagster_get_run_logs","dagster_launch_run","dagster_list_assets","dagster_list_jobs","dagster_list_runs","dagster_list_schedules","dagster_list_sensors","dagster_materialize_assets","dagster_reexecute_run","dagster_report_asset_materialization","dagster_start_schedule","dagster_start_sensor","dagster_stop_schedule","dagster_stop_sensor","dagster_terminate_run","dagster_wipe_asset","databricks_cancel_run","databricks_execute_sql","databricks_get_cluster","databricks_get_job","databricks_get_run","databricks_get_run_output","databricks_get_statement","databricks_list_clusters","databricks_list_jobs","databricks_list_runs","databricks_list_warehouses","databricks_run_job","datadog_add_incident_todo","datadog_cancel_downtime","datadog_create_dashboard","datadog_create_downtime","datadog_create_event","datadog_create_incident","datadog_create_monitor","datadog_create_slo","datadog_delete_dashboard","datadog_delete_slo","datadog_get_browser_synthetics_results","datadog_get_dashboard","datadog_get_incident","datadog_get_monitor","datadog_get_security_signal","datadog_get_slo","datadog_get_slo_history","datadog_get_synthetics_results","datadog_get_synthetics_test","datadog_list_dashboards","datadog_list_downtimes","datadog_list_incidents","datadog_list_monitors","datadog_list_security_rules","datadog_list_security_signals","datadog_list_services","datadog_list_slos","datadog_list_synthetics_tests","datadog_mute_monitor","datadog_query_logs","datadog_query_timeseries","datadog_search_spans","datadog_send_logs","datadog_submit_metrics","datadog_trigger_synthetics_tests","datadog_unmute_monitor","datadog_update_incident","datadog_update_security_signal_assignee","datadog_update_security_signal_state","datadog_update_slo","datadog_update_synthetics_status","datagma_enrich_company","datagma_enrich_person","datagma_find_email","datagma_find_phone","datagma_get_credits","daytona_create_sandbox","daytona_delete_sandbox","daytona_download_file","daytona_execute_command","daytona_get_sandbox","daytona_git_clone","daytona_list_files","daytona_list_sandboxes","daytona_run_code","daytona_start_sandbox","daytona_stop_sandbox","daytona_upload_file","deployed_block_executor","deployments_deploy","deployments_get_version","deployments_list_versions","deployments_promote","deployments_undeploy","devin_append_session_tags","devin_archive_session","devin_create_session","devin_get_session","devin_get_session_tags","devin_list_session_attachments","devin_list_session_messages","devin_list_sessions","devin_replace_session_tags","devin_send_message","devin_terminate_session","discord_add_reaction","discord_archive_thread","discord_assign_role","discord_ban_member","discord_bulk_delete_messages","discord_create_channel","discord_create_invite","discord_create_role","discord_create_thread","discord_create_webhook","discord_delete_channel","discord_delete_invite","discord_delete_message","discord_delete_role","discord_delete_webhook","discord_edit_message","discord_execute_webhook","discord_get_channel","discord_get_invite","discord_get_member","discord_get_messages","discord_get_pinned_messages","discord_get_server","discord_get_user","discord_get_webhook","discord_join_thread","discord_kick_member","discord_leave_thread","discord_list_channels","discord_list_roles","discord_pin_message","discord_remove_reaction","discord_remove_role","discord_send_message","discord_unban_member","discord_unpin_message","discord_update_channel","discord_update_member","discord_update_role","docusign_create_from_template","docusign_download_document","docusign_get_envelope","docusign_list_envelopes","docusign_list_recipients","docusign_list_templates","docusign_send_envelope","docusign_void_envelope","downdetector_get_company","downdetector_get_company_attribution","downdetector_get_company_baseline","downdetector_get_company_events","downdetector_get_company_incidents","downdetector_get_company_indicators","downdetector_get_company_last_15","downdetector_get_company_status","downdetector_get_provider","downdetector_get_reports","downdetector_get_site_companies","downdetector_list_categories","downdetector_list_incidents","downdetector_list_sites","downdetector_search_companies","dropbox_copy","dropbox_create_folder","dropbox_create_shared_link","dropbox_delete","dropbox_download","dropbox_get_metadata","dropbox_list_folder","dropbox_list_revisions","dropbox_list_shared_links","dropbox_move","dropbox_restore","dropbox_search","dropbox_upload","dropcontact_enrich_contact","dspy_chain_of_thought","dspy_predict","dspy_react","dub_bulk_create_links","dub_bulk_delete_links","dub_bulk_update_links","dub_create_link","dub_create_tag","dub_delete_link","dub_get_analytics","dub_get_events","dub_get_link","dub_get_links_count","dub_get_qr_code","dub_list_domains","dub_list_folders","dub_list_links","dub_list_tags","dub_update_link","dub_upsert_link","duckduckgo_search","dynamodb_delete","dynamodb_get","dynamodb_introspect","dynamodb_put","dynamodb_query","dynamodb_scan","dynamodb_update","dynatrace_add_problem_comment","dynatrace_add_tags","dynatrace_close_problem","dynatrace_create_settings_object","dynatrace_create_slo","dynatrace_delete_problem_comment","dynatrace_delete_settings_object","dynatrace_delete_slo","dynatrace_delete_tag","dynatrace_execute_synthetic_monitors","dynatrace_get_attack","dynatrace_get_audit_logs","dynatrace_get_entity","dynatrace_get_event","dynatrace_get_metric","dynatrace_get_problem","dynatrace_get_problem_comment","dynatrace_get_security_problem","dynatrace_get_settings_object","dynatrace_get_slo","dynatrace_get_synthetic_batch","dynatrace_ingest_event","dynatrace_ingest_logs","dynatrace_ingest_metrics","dynatrace_list_attacks","dynatrace_list_entities","dynatrace_list_entity_types","dynatrace_list_events","dynatrace_list_metrics","dynatrace_list_problem_comments","dynatrace_list_problems","dynatrace_list_remediation_items","dynatrace_list_security_problems","dynatrace_list_settings_objects","dynatrace_list_settings_schemas","dynatrace_list_slos","dynatrace_list_synthetic_monitors","dynatrace_list_tags","dynatrace_mute_security_problem","dynatrace_mute_security_problems","dynatrace_query_metrics","dynatrace_search_logs","dynatrace_unmute_security_problem","dynatrace_unmute_security_problems","dynatrace_update_problem_comment","dynatrace_update_settings_object","dynatrace_update_slo","elasticsearch_bulk","elasticsearch_cluster_health","elasticsearch_cluster_stats","elasticsearch_count","elasticsearch_create_index","elasticsearch_delete_document","elasticsearch_delete_index","elasticsearch_get_document","elasticsearch_get_index","elasticsearch_index_document","elasticsearch_list_indices","elasticsearch_search","elasticsearch_update_document","elevenlabs_audio_isolation","elevenlabs_edit_voice_settings","elevenlabs_get_user","elevenlabs_get_voice","elevenlabs_get_voice_settings","elevenlabs_list_models","elevenlabs_list_voices","elevenlabs_sound_effects","elevenlabs_speech_to_speech","elevenlabs_tts","emailbison_attach_leads_to_campaign","emailbison_attach_tags_to_leads","emailbison_create_campaign","emailbison_create_lead","emailbison_create_tag","emailbison_get_lead","emailbison_list_campaigns","emailbison_list_leads","emailbison_list_replies","emailbison_list_tags","emailbison_update_campaign","emailbison_update_campaign_status","emailbison_update_lead","embeddings_cohere","embeddings_gemini","embeddings_mistral","embeddings_openai","embeddings_openrouter","enrich_check_credits","enrich_company_funding","enrich_company_lookup","enrich_company_revenue","enrich_disposable_email_check","enrich_email_to_ip","enrich_email_to_person_lite","enrich_email_to_phone","enrich_email_to_profile","enrich_find_email","enrich_get_post_details","enrich_ip_to_company","enrich_linkedin_profile","enrich_linkedin_to_personal_email","enrich_linkedin_to_work_email","enrich_phone_finder","enrich_reverse_hash_lookup","enrich_sales_pointer_people","enrich_search_company","enrich_search_company_activities","enrich_search_company_employees","enrich_search_jobs","enrich_search_logo","enrich_search_people","enrich_search_people_activities","enrich_search_post_comments","enrich_search_post_comments_by_url","enrich_search_post_reactions","enrich_search_post_reactions_by_url","enrich_search_posts","enrich_search_similar_companies","enrich_verify_email","enrichment_run","enrow_find_email","enrow_verify_email","exa_agent","exa_answer","exa_find_similar_links","exa_get_contents","exa_search","extend_parser","extend_parser_v2","fathom_get_summary","fathom_get_transcript","fathom_list_meeting_types","fathom_list_meetings","fathom_list_team_members","fathom_list_teams","file_append","file_compress","file_decompress","file_fetch","file_get","file_get_content","file_manage_sharing","file_parser","file_parser_v2","file_parser_v3","file_read","file_write","findymail_find_email_from_linkedin","findymail_find_email_from_name","findymail_find_emails_by_domain","findymail_find_employees","findymail_find_phone","findymail_get_company","findymail_get_credits","findymail_lookup_technologies","findymail_reverse_email_lookup","findymail_search_technologies","findymail_verify_email","firecrawl_agent","firecrawl_batch_scrape","firecrawl_batch_scrape_status","firecrawl_cancel_crawl","firecrawl_crawl","firecrawl_crawl_status","firecrawl_credit_usage","firecrawl_extract","firecrawl_extract_status","firecrawl_map","firecrawl_parse","firecrawl_scrape","firecrawl_search","fireflies_add_to_live_meeting","fireflies_create_bite","fireflies_delete_transcript","fireflies_get_transcript","fireflies_get_user","fireflies_list_bites","fireflies_list_contacts","fireflies_list_transcripts","fireflies_list_users","fireflies_upload_audio","flint_create_task","flint_generate_pages","flint_get_task","function_execute","gamma_check_status","gamma_generate","gamma_generate_from_template","gamma_list_folders","gamma_list_themes","github_add_assignees","github_add_assignees_v2","github_add_labels","github_add_labels_v2","github_cancel_workflow_run","github_cancel_workflow_run_v2","github_check_star","github_check_star_v2","github_close_issue","github_close_issue_v2","github_close_pr","github_close_pr_v2","github_comment","github_comment_v2","github_compare_commits","github_compare_commits_v2","github_create_branch","github_create_branch_v2","github_create_comment_reaction","github_create_comment_reaction_v2","github_create_file","github_create_file_v2","github_create_gist","github_create_gist_v2","github_create_issue","github_create_issue_reaction","github_create_issue_reaction_v2","github_create_issue_v2","github_create_milestone","github_create_milestone_v2","github_create_pr","github_create_pr_review","github_create_pr_review_v2","github_create_pr_v2","github_create_project","github_create_project_v2","github_create_release","github_create_release_v2","github_delete_branch","github_delete_branch_v2","github_delete_comment","github_delete_comment_reaction","github_delete_comment_reaction_v2","github_delete_comment_v2","github_delete_file","github_delete_file_v2","github_delete_gist","github_delete_gist_v2","github_delete_issue_reaction","github_delete_issue_reaction_v2","github_delete_milestone","github_delete_milestone_v2","github_delete_project","github_delete_project_v2","github_delete_release","github_delete_release_v2","github_fork_gist","github_fork_gist_v2","github_fork_repo","github_fork_repo_v2","github_get_branch","github_get_branch_protection","github_get_branch_protection_v2","github_get_branch_v2","github_get_commit","github_get_commit_v2","github_get_file_content","github_get_file_content_v2","github_get_gist","github_get_gist_v2","github_get_issue","github_get_issue_v2","github_get_latest_release","github_get_latest_release_v2","github_get_milestone","github_get_milestone_v2","github_get_pr_files","github_get_pr_files_v2","github_get_project","github_get_project_v2","github_get_readme","github_get_readme_v2","github_get_release","github_get_release_v2","github_get_tree","github_get_tree_v2","github_get_workflow","github_get_workflow_run","github_get_workflow_run_v2","github_get_workflow_v2","github_issue_comment","github_issue_comment_v2","github_job_logs","github_latest_commit","github_latest_commit_v2","github_list_branches","github_list_branches_v2","github_list_commits","github_list_commits_v2","github_list_forks","github_list_forks_v2","github_list_gists","github_list_gists_v2","github_list_issue_comments","github_list_issue_comments_v2","github_list_issues","github_list_issues_v2","github_list_milestones","github_list_milestones_v2","github_list_pr_comments","github_list_pr_comments_v2","github_list_projects","github_list_projects_v2","github_list_prs","github_list_prs_v2","github_list_releases","github_list_releases_v2","github_list_review_threads","github_list_stargazers","github_list_stargazers_v2","github_list_tags","github_list_tags_v2","github_list_workflow_runs","github_list_workflow_runs_v2","github_list_workflows","github_list_workflows_v2","github_merge_pr","github_merge_pr_v2","github_pr","github_pr_v2","github_remove_label","github_remove_label_v2","github_reply_review_thread","github_repo_info","github_repo_info_v2","github_request_reviewers","github_request_reviewers_v2","github_rerun_workflow","github_rerun_workflow_v2","github_resolve_review_thread","github_search_code","github_search_code_v2","github_search_commits","github_search_commits_v2","github_search_issues","github_search_issues_v2","github_search_repos","github_search_repos_v2","github_search_users","github_search_users_v2","github_star_gist","github_star_gist_v2","github_star_repo","github_star_repo_v2","github_status_check_rollup","github_trigger_workflow","github_trigger_workflow_v2","github_unstar_gist","github_unstar_gist_v2","github_unstar_repo","github_unstar_repo_v2","github_update_branch_protection","github_update_branch_protection_v2","github_update_comment","github_update_comment_v2","github_update_file","github_update_file_v2","github_update_gist","github_update_gist_v2","github_update_issue","github_update_issue_v2","github_update_milestone","github_update_milestone_v2","github_update_pr","github_update_pr_v2","github_update_project","github_update_project_v2","github_update_release","github_update_release_v2","gitlab_activate_user","gitlab_add_member","gitlab_add_saml_group_link","gitlab_approve_access_request","gitlab_approve_merge_request","gitlab_approve_user","gitlab_ban_user","gitlab_block_user","gitlab_cancel_pipeline","gitlab_compare_branches","gitlab_create_branch","gitlab_create_file","gitlab_create_issue","gitlab_create_issue_note","gitlab_create_merge_request","gitlab_create_merge_request_note","gitlab_create_pipeline","gitlab_create_release","gitlab_create_user","gitlab_deactivate_user","gitlab_delete_branch","gitlab_delete_issue","gitlab_delete_saml_group_link","gitlab_delete_user","gitlab_delete_user_identity","gitlab_deny_access_request","gitlab_get_file","gitlab_get_group","gitlab_get_issue","gitlab_get_job_log","gitlab_get_merge_request","gitlab_get_merge_request_changes","gitlab_get_pipeline","gitlab_get_project","gitlab_invite_member","gitlab_list_access_requests","gitlab_list_branches","gitlab_list_commits","gitlab_list_groups","gitlab_list_invitations","gitlab_list_issues","gitlab_list_members","gitlab_list_merge_requests","gitlab_list_pipeline_jobs","gitlab_list_pipelines","gitlab_list_projects","gitlab_list_releases","gitlab_list_repository_tree","gitlab_list_saml_group_links","gitlab_list_user_memberships","gitlab_merge_merge_request","gitlab_play_job","gitlab_reject_user","gitlab_remove_member","gitlab_retry_pipeline","gitlab_revoke_invitation","gitlab_search_users","gitlab_unban_user","gitlab_unblock_user","gitlab_update_file","gitlab_update_invitation","gitlab_update_issue","gitlab_update_member","gitlab_update_merge_request","gitlab_update_user","gmail_add_label","gmail_add_label_v2","gmail_archive","gmail_archive_v2","gmail_create_label_v2","gmail_delete","gmail_delete_draft_v2","gmail_delete_label_v2","gmail_delete_v2","gmail_draft","gmail_draft_v2","gmail_edit_draft_v2","gmail_get_draft_v2","gmail_get_thread_v2","gmail_list_drafts_v2","gmail_list_labels_v2","gmail_list_threads_v2","gmail_mark_read","gmail_mark_read_v2","gmail_mark_unread","gmail_mark_unread_v2","gmail_move","gmail_move_v2","gmail_read","gmail_read_v2","gmail_remove_label","gmail_remove_label_v2","gmail_search","gmail_search_v2","gmail_send","gmail_send_v2","gmail_trash_thread_v2","gmail_unarchive","gmail_unarchive_v2","gmail_untrash_thread_v2","gmail_update_label_v2","gong_aggregate_activity","gong_aggregate_by_period","gong_answered_scorecards","gong_ask_anything","gong_assign_flow_prospects","gong_create_call","gong_day_by_day_activity","gong_get_brief","gong_get_call","gong_get_call_transcript","gong_get_coaching","gong_get_extensive_calls","gong_get_folder_content","gong_get_logs","gong_get_prospect_flows","gong_get_user","gong_interaction_stats","gong_list_calls","gong_list_flows","gong_list_library_folders","gong_list_scorecards","gong_list_trackers","gong_list_users","gong_list_workspaces","gong_lookup_email","gong_lookup_phone","gong_purge_email_address","gong_purge_phone_number","gong_unassign_flow_prospects","google_ads_ad_performance","google_ads_campaign_performance","google_ads_list_ad_groups","google_ads_list_campaigns","google_ads_list_customers","google_ads_search","google_appsheet_add_rows","google_appsheet_delete_rows","google_appsheet_edit_rows","google_appsheet_find_rows","google_bigquery_create_dataset","google_bigquery_create_table","google_bigquery_delete_dataset","google_bigquery_delete_table","google_bigquery_get_query_results","google_bigquery_get_table","google_bigquery_insert_rows","google_bigquery_list_datasets","google_bigquery_list_table_data","google_bigquery_list_tables","google_bigquery_query","google_books_volume_details","google_books_volume_search","google_calendar_create","google_calendar_create_calendar","google_calendar_create_calendar_v2","google_calendar_create_v2","google_calendar_delete","google_calendar_delete_calendar","google_calendar_delete_calendar_v2","google_calendar_delete_v2","google_calendar_freebusy","google_calendar_freebusy_v2","google_calendar_get","google_calendar_get_v2","google_calendar_instances","google_calendar_instances_v2","google_calendar_invite","google_calendar_invite_v2","google_calendar_list","google_calendar_list_acl","google_calendar_list_acl_v2","google_calendar_list_calendars","google_calendar_list_calendars_v2","google_calendar_list_v2","google_calendar_move","google_calendar_move_v2","google_calendar_quick_add","google_calendar_quick_add_v2","google_calendar_share_calendar","google_calendar_share_calendar_v2","google_calendar_unshare_calendar","google_calendar_unshare_calendar_v2","google_calendar_update","google_calendar_update_acl","google_calendar_update_acl_v2","google_calendar_update_calendar","google_calendar_update_calendar_v2","google_calendar_update_v2","google_contacts_create","google_contacts_delete","google_contacts_get","google_contacts_list","google_contacts_search","google_contacts_update","google_docs_create","google_docs_create_named_range","google_docs_create_paragraph_bullets","google_docs_delete_content_range","google_docs_delete_named_range","google_docs_delete_paragraph_bullets","google_docs_insert_image","google_docs_insert_page_break","google_docs_insert_table","google_docs_insert_text","google_docs_read","google_docs_replace_text","google_docs_update_paragraph_style","google_docs_update_text_style","google_docs_write","google_drive_copy","google_drive_create_comment","google_drive_create_folder","google_drive_delete","google_drive_delete_comment","google_drive_download","google_drive_export","google_drive_get_about","google_drive_get_content","google_drive_get_file","google_drive_get_revision","google_drive_list","google_drive_list_comments","google_drive_list_permissions","google_drive_list_revisions","google_drive_move","google_drive_search","google_drive_share","google_drive_trash","google_drive_unshare","google_drive_untrash","google_drive_update","google_drive_upload","google_forms_batch_update","google_forms_create_form","google_forms_create_watch","google_forms_delete_watch","google_forms_get_form","google_forms_get_responses","google_forms_list_watches","google_forms_renew_watch","google_forms_set_publish_settings","google_groups_add_alias","google_groups_add_member","google_groups_create_group","google_groups_delete_group","google_groups_get_group","google_groups_get_member","google_groups_get_settings","google_groups_has_member","google_groups_list_aliases","google_groups_list_groups","google_groups_list_members","google_groups_remove_alias","google_groups_remove_member","google_groups_update_group","google_groups_update_member","google_groups_update_settings","google_maps_air_quality","google_maps_directions","google_maps_distance_matrix","google_maps_elevation","google_maps_geocode","google_maps_geolocate","google_maps_place_details","google_maps_places_nearby","google_maps_places_search","google_maps_pollen","google_maps_reverse_geocode","google_maps_snap_to_roads","google_maps_solar","google_maps_speed_limits","google_maps_timezone","google_maps_validate_address","google_meet_create_space","google_meet_end_conference","google_meet_get_conference_record","google_meet_get_space","google_meet_list_conference_records","google_meet_list_participants","google_pagespeed_analyze","google_search","google_sheets_append","google_sheets_append_v2","google_sheets_batch_clear_v2","google_sheets_batch_get_v2","google_sheets_batch_update_v2","google_sheets_clear_v2","google_sheets_copy_sheet_v2","google_sheets_create_spreadsheet_v2","google_sheets_delete_rows_v2","google_sheets_delete_sheet_v2","google_sheets_delete_spreadsheet_v2","google_sheets_get_spreadsheet_v2","google_sheets_read","google_sheets_read_v2","google_sheets_update","google_sheets_update_v2","google_sheets_write","google_sheets_write_v2","google_slides_add_image","google_slides_add_slide","google_slides_batch_update","google_slides_copy_presentation","google_slides_create","google_slides_create_line","google_slides_create_paragraph_bullets","google_slides_create_shape","google_slides_create_sheets_chart","google_slides_create_table","google_slides_create_video","google_slides_delete_object","google_slides_delete_paragraph_bullets","google_slides_delete_table_column","google_slides_delete_table_row","google_slides_delete_text","google_slides_duplicate_object","google_slides_export_presentation","google_slides_get_page","google_slides_get_thumbnail","google_slides_group_objects","google_slides_insert_table_columns","google_slides_insert_table_rows","google_slides_insert_text","google_slides_merge_table_cells","google_slides_read","google_slides_refresh_sheets_chart","google_slides_replace_all_shapes_with_image","google_slides_replace_all_shapes_with_sheets_chart","google_slides_replace_all_text","google_slides_replace_image","google_slides_reroute_line","google_slides_ungroup_objects","google_slides_unmerge_table_cells","google_slides_update_image_properties","google_slides_update_line_category","google_slides_update_line_properties","google_slides_update_page_element_alt_text","google_slides_update_page_element_transform","google_slides_update_page_elements_z_order","google_slides_update_page_properties","google_slides_update_paragraph_style","google_slides_update_shape_properties","google_slides_update_slide_properties","google_slides_update_slides_position","google_slides_update_table_border_properties","google_slides_update_table_cell_properties","google_slides_update_table_column_properties","google_slides_update_table_row_properties","google_slides_update_text_style","google_slides_update_video_properties","google_slides_write","google_tasks_create","google_tasks_delete","google_tasks_get","google_tasks_list","google_tasks_list_task_lists","google_tasks_update","google_translate_detect","google_translate_text","google_vault_add_held_accounts","google_vault_add_matters_permissions","google_vault_close_matters","google_vault_create_matters","google_vault_create_matters_export","google_vault_create_matters_holds","google_vault_create_saved_query","google_vault_delete_matters","google_vault_delete_matters_export","google_vault_delete_matters_holds","google_vault_delete_saved_query","google_vault_download_export_file","google_vault_list_matters","google_vault_list_matters_export","google_vault_list_matters_holds","google_vault_list_saved_queries","google_vault_remove_held_accounts","google_vault_remove_matters_permissions","google_vault_reopen_matters","google_vault_undelete_matters","google_vault_update_matters","google_vault_update_matters_holds","grafana_check_data_source_health","grafana_create_alert_rule","grafana_create_annotation","grafana_create_contact_point","grafana_create_dashboard","grafana_create_folder","grafana_delete_alert_rule","grafana_delete_annotation","grafana_delete_contact_point","grafana_delete_dashboard","grafana_delete_folder","grafana_get_alert_rule","grafana_get_alert_rule_group","grafana_get_dashboard","grafana_get_data_source","grafana_get_folder","grafana_get_health","grafana_list_alert_rules","grafana_list_annotations","grafana_list_contact_points","grafana_list_dashboards","grafana_list_data_sources","grafana_list_folders","grafana_move_folder","grafana_query_data_source","grafana_update_alert_rule","grafana_update_annotation","grafana_update_contact_point","grafana_update_dashboard","grafana_update_folder","grain_create_hook","grain_create_hook_v2","grain_delete_hook","grain_delete_hook_v2","grain_get_recording","grain_get_transcript","grain_list_hooks","grain_list_hooks_v2","grain_list_meeting_types","grain_list_recordings","grain_list_teams","grain_list_views","granola_get_note","granola_list_folders","granola_list_notes","greenhouse_get_application","greenhouse_get_candidate","greenhouse_get_job","greenhouse_get_user","greenhouse_list_applications","greenhouse_list_candidates","greenhouse_list_departments","greenhouse_list_job_stages","greenhouse_list_jobs","greenhouse_list_offices","greenhouse_list_users","greptile_index_repo","greptile_query","greptile_search","greptile_status","guardrails_validate","hex_cancel_run","hex_create_collection","hex_create_group","hex_deactivate_user","hex_delete_group","hex_get_collection","hex_get_data_connection","hex_get_group","hex_get_project","hex_get_project_runs","hex_get_queried_tables","hex_get_run_status","hex_list_collections","hex_list_data_connections","hex_list_groups","hex_list_projects","hex_list_users","hex_run_project","hex_update_collection","hex_update_group","hex_update_project","http_request","hubspot_add_list_memberships","hubspot_create_appointment","hubspot_create_association","hubspot_create_company","hubspot_create_contact","hubspot_create_deal","hubspot_create_email","hubspot_create_line_item","hubspot_create_list","hubspot_create_note","hubspot_create_ticket","hubspot_delete_association","hubspot_delete_company","hubspot_delete_contact","hubspot_delete_deal","hubspot_delete_line_item","hubspot_delete_ticket","hubspot_get_appointment","hubspot_get_association_labels","hubspot_get_cart","hubspot_get_company","hubspot_get_contact","hubspot_get_deal","hubspot_get_email","hubspot_get_line_item","hubspot_get_list","hubspot_get_list_memberships","hubspot_get_marketing_event","hubspot_get_note","hubspot_get_properties","hubspot_get_quote","hubspot_get_ticket","hubspot_get_users","hubspot_list_appointments","hubspot_list_associations","hubspot_list_carts","hubspot_list_companies","hubspot_list_contacts","hubspot_list_deals","hubspot_list_emails","hubspot_list_line_items","hubspot_list_lists","hubspot_list_marketing_events","hubspot_list_notes","hubspot_list_owners","hubspot_list_quotes","hubspot_list_tickets","hubspot_remove_list_memberships","hubspot_search_companies","hubspot_search_contacts","hubspot_search_deals","hubspot_search_emails","hubspot_search_line_items","hubspot_search_notes","hubspot_search_quotes","hubspot_search_tickets","hubspot_update_appointment","hubspot_update_company","hubspot_update_contact","hubspot_update_deal","hubspot_update_line_item","hubspot_update_ticket","huggingface_chat","hunter_companies_find","hunter_discover","hunter_domain_search","hunter_email_count","hunter_email_finder","hunter_email_verifier","iam_add_user_to_group","iam_attach_role_policy","iam_attach_user_policy","iam_create_access_key","iam_create_role","iam_create_user","iam_delete_access_key","iam_delete_role","iam_delete_user","iam_detach_role_policy","iam_detach_user_policy","iam_get_role","iam_get_user","iam_list_attached_role_policies","iam_list_attached_user_policies","iam_list_groups","iam_list_policies","iam_list_roles","iam_list_users","iam_remove_user_from_group","iam_simulate_principal_policy","icypeas_find_email","icypeas_verify_email","identity_center_check_assignment_deletion_status","identity_center_check_assignment_status","identity_center_create_account_assignment","identity_center_delete_account_assignment","identity_center_describe_account","identity_center_get_group","identity_center_get_user","identity_center_list_account_assignments","identity_center_list_accounts","identity_center_list_groups","identity_center_list_instances","identity_center_list_permission_sets","image_generate","incidentio_actions_create","incidentio_actions_list","incidentio_actions_show","incidentio_actions_update","incidentio_alert_events_create","incidentio_alerts_list","incidentio_alerts_resolve","incidentio_alerts_show","incidentio_catalog_entries_list","incidentio_catalog_types_list","incidentio_custom_fields_create","incidentio_custom_fields_delete","incidentio_custom_fields_list","incidentio_custom_fields_show","incidentio_custom_fields_update","incidentio_escalation_paths_create","incidentio_escalation_paths_delete","incidentio_escalation_paths_list","incidentio_escalation_paths_show","incidentio_escalation_paths_update","incidentio_escalations_cancel","incidentio_escalations_create","incidentio_escalations_list","incidentio_escalations_show","incidentio_follow_ups_create","incidentio_follow_ups_list","incidentio_follow_ups_show","incidentio_follow_ups_update","incidentio_incident_alerts_list","incidentio_incident_memberships_create","incidentio_incident_memberships_revoke","incidentio_incident_participants_list","incidentio_incident_roles_create","incidentio_incident_roles_delete","incidentio_incident_roles_list","incidentio_incident_roles_show","incidentio_incident_roles_update","incidentio_incident_statuses_list","incidentio_incident_timestamps_list","incidentio_incident_timestamps_show","incidentio_incident_types_list","incidentio_incident_updates_list","incidentio_incidents_create","incidentio_incidents_list","incidentio_incidents_show","incidentio_incidents_update","incidentio_on_call_now","incidentio_schedule_entries_list","incidentio_schedule_overrides_create","incidentio_schedule_overrides_list","incidentio_schedules_create","incidentio_schedules_delete","incidentio_schedules_list","incidentio_schedules_show","incidentio_schedules_update","incidentio_severities_list","incidentio_teams_list","incidentio_teams_show","incidentio_users_list","incidentio_users_show","incidentio_workflows_create","incidentio_workflows_delete","incidentio_workflows_list","incidentio_workflows_show","incidentio_workflows_update","infisical_create_secret","infisical_delete_secret","infisical_get_secret","infisical_list_secrets","infisical_update_secret","instagram_delete_comment","instagram_download_media","instagram_get_account_insights","instagram_get_container_status","instagram_get_conversation_messages","instagram_get_media","instagram_get_media_insights","instagram_get_message","instagram_get_profile","instagram_get_publishing_limit","instagram_hide_comment","instagram_list_comments","instagram_list_conversations","instagram_list_media","instagram_list_stories","instagram_private_reply","instagram_publish_carousel","instagram_publish_image","instagram_publish_reel","instagram_publish_story","instagram_publish_video","instagram_reply_to_comment","instagram_send_text_message","instagram_set_comments_enabled","instantly_activate_campaign","instantly_create_campaign","instantly_create_lead","instantly_create_lead_list","instantly_delete_campaign","instantly_delete_leads","instantly_get_lead","instantly_list_campaigns","instantly_list_emails","instantly_list_lead_lists","instantly_list_leads","instantly_patch_campaign","instantly_patch_lead","instantly_pause_campaign","instantly_reply_to_email","instantly_update_lead_interest_status","intercom_assign_conversation_v2","intercom_attach_contact_to_company_v2","intercom_close_conversation_v2","intercom_create_company","intercom_create_company_v2","intercom_create_contact","intercom_create_contact_v2","intercom_create_event_v2","intercom_create_message","intercom_create_message_v2","intercom_create_note_v2","intercom_create_tag_v2","intercom_create_ticket","intercom_create_ticket_v2","intercom_delete_contact","intercom_delete_contact_v2","intercom_detach_contact_from_company_v2","intercom_get_company","intercom_get_company_v2","intercom_get_contact","intercom_get_contact_v2","intercom_get_conversation","intercom_get_conversation_v2","intercom_get_ticket","intercom_get_ticket_v2","intercom_list_admins_v2","intercom_list_companies","intercom_list_companies_v2","intercom_list_contacts","intercom_list_contacts_v2","intercom_list_conversations","intercom_list_conversations_v2","intercom_list_tags_v2","intercom_open_conversation_v2","intercom_reply_conversation","intercom_reply_conversation_v2","intercom_search_contacts","intercom_search_contacts_v2","intercom_search_conversations","intercom_search_conversations_v2","intercom_snooze_conversation_v2","intercom_tag_contact_v2","intercom_tag_conversation_v2","intercom_untag_contact_v2","intercom_update_contact","intercom_update_contact_v2","intercom_update_ticket_v2","jina_read_url","jina_search","jira_add_attachment","jira_add_comment","jira_add_watcher","jira_add_worklog","jira_assign_issue","jira_bulk_read","jira_create_issue_link","jira_delete_attachment","jira_delete_comment","jira_delete_issue","jira_delete_issue_link","jira_delete_worklog","jira_get_attachments","jira_get_comments","jira_get_fields","jira_get_project","jira_get_transitions","jira_get_users","jira_get_worklogs","jira_list_issue_types","jira_list_projects","jira_remove_watcher","jira_retrieve","jira_search_issues","jira_search_users","jira_transition_issue","jira_update","jira_update_comment","jira_update_worklog","jira_write","jsm_add_comment","jsm_add_customer","jsm_add_organization","jsm_add_participants","jsm_answer_approval","jsm_attach_form","jsm_copy_forms","jsm_create_object","jsm_create_organization","jsm_create_request","jsm_delete_form","jsm_delete_object","jsm_externalise_form","jsm_get_approvals","jsm_get_comments","jsm_get_customers","jsm_get_form","jsm_get_form_answers","jsm_get_form_structure","jsm_get_form_templates","jsm_get_issue_forms","jsm_get_object","jsm_get_object_schema","jsm_get_object_type_attributes","jsm_get_organizations","jsm_get_participants","jsm_get_queues","jsm_get_request","jsm_get_request_type_fields","jsm_get_request_types","jsm_get_requests","jsm_get_service_desks","jsm_get_sla","jsm_get_transitions","jsm_internalise_form","jsm_list_object_schemas","jsm_list_object_types","jsm_reopen_form","jsm_save_form_answers","jsm_search_objects_aql","jsm_submit_form","jsm_transition_request","jsm_update_object","jupyter_copy_content","jupyter_create_file","jupyter_create_session","jupyter_delete_content","jupyter_delete_session","jupyter_get_content","jupyter_interrupt_kernel","jupyter_list_contents","jupyter_list_kernels","jupyter_list_kernelspecs","jupyter_list_sessions","jupyter_rename_content","jupyter_restart_kernel","jupyter_start_kernel","jupyter_stop_kernel","jupyter_upload_file","kalshi_amend_order","kalshi_amend_order_v2","kalshi_cancel_order","kalshi_cancel_order_v2","kalshi_create_order","kalshi_create_order_v2","kalshi_get_balance","kalshi_get_balance_v2","kalshi_get_candlesticks","kalshi_get_candlesticks_v2","kalshi_get_event","kalshi_get_event_candlesticks","kalshi_get_event_candlesticks_v2","kalshi_get_event_v2","kalshi_get_events","kalshi_get_events_v2","kalshi_get_exchange_announcements","kalshi_get_exchange_announcements_v2","kalshi_get_exchange_schedule","kalshi_get_exchange_schedule_v2","kalshi_get_exchange_status","kalshi_get_exchange_status_v2","kalshi_get_fills","kalshi_get_fills_v2","kalshi_get_market","kalshi_get_market_v2","kalshi_get_markets","kalshi_get_markets_v2","kalshi_get_order","kalshi_get_order_v2","kalshi_get_orderbook","kalshi_get_orderbook_v2","kalshi_get_orders","kalshi_get_orders_v2","kalshi_get_positions","kalshi_get_positions_v2","kalshi_get_series_by_ticker","kalshi_get_series_by_ticker_v2","kalshi_get_series_list","kalshi_get_series_list_v2","kalshi_get_settlements","kalshi_get_settlements_v2","kalshi_get_trades","kalshi_get_trades_v2","ketch_get_consent","ketch_get_subscriptions","ketch_invoke_right","ketch_set_consent","ketch_set_subscriptions","knowledge_create_document","knowledge_delete_chunk","knowledge_delete_document","knowledge_get_connector","knowledge_get_document","knowledge_list_chunks","knowledge_list_connectors","knowledge_list_documents","knowledge_list_tags","knowledge_search","knowledge_trigger_sync","knowledge_update_chunk","knowledge_upload_chunk","knowledge_upsert_document","langsmith_create_feedback","langsmith_create_run","langsmith_create_runs_batch","langsmith_get_run","langsmith_update_run","latex_compile","latex_get_package","latex_list_fonts","latex_search_packages","launchdarkly_create_flag","launchdarkly_delete_flag","launchdarkly_get_audit_log","launchdarkly_get_flag","launchdarkly_get_flag_status","launchdarkly_list_environments","launchdarkly_list_flags","launchdarkly_list_members","launchdarkly_list_projects","launchdarkly_list_segments","launchdarkly_toggle_flag","launchdarkly_update_flag","leadmagic_company_search","leadmagic_email_to_profile","leadmagic_find_email","leadmagic_find_mobile","leadmagic_get_credits","leadmagic_profile_search","leadmagic_profile_to_email","leadmagic_role_finder","leadmagic_validate_email","lemlist_get_activities","lemlist_get_lead","lemlist_send_email","linear_add_label_to_issue","linear_add_label_to_project","linear_archive_issue","linear_archive_label","linear_archive_project","linear_create_attachment","linear_create_comment","linear_create_customer","linear_create_customer_request","linear_create_customer_status","linear_create_customer_tier","linear_create_cycle","linear_create_favorite","linear_create_issue","linear_create_issue_relation","linear_create_label","linear_create_project","linear_create_project_label","linear_create_project_milestone","linear_create_project_status","linear_create_project_update","linear_create_workflow_state","linear_delete_attachment","linear_delete_comment","linear_delete_customer","linear_delete_customer_status","linear_delete_customer_tier","linear_delete_issue","linear_delete_issue_relation","linear_delete_project","linear_delete_project_label","linear_delete_project_milestone","linear_delete_project_status","linear_get_active_cycle","linear_get_customer","linear_get_cycle","linear_get_issue","linear_get_project","linear_get_viewer","linear_list_attachments","linear_list_comments","linear_list_customer_requests","linear_list_customer_statuses","linear_list_customer_tiers","linear_list_customers","linear_list_cycles","linear_list_favorites","linear_list_issue_relations","linear_list_labels","linear_list_notifications","linear_list_project_labels","linear_list_project_milestones","linear_list_project_statuses","linear_list_project_updates","linear_list_projects","linear_list_teams","linear_list_users","linear_list_workflow_states","linear_merge_customers","linear_read_issues","linear_remove_label_from_issue","linear_remove_label_from_project","linear_search_issues","linear_unarchive_issue","linear_update_attachment","linear_update_comment","linear_update_customer","linear_update_customer_request","linear_update_customer_status","linear_update_customer_tier","linear_update_issue","linear_update_label","linear_update_notification","linear_update_project","linear_update_project_label","linear_update_project_milestone","linear_update_project_status","linear_update_workflow_state","linkedin_get_profile","linkedin_share_post","linkup_search","linq_add_participant","linq_check_imessage","linq_check_rcs","linq_create_attachment","linq_create_chat","linq_create_contact_card","linq_create_webhook_subscription","linq_delete_attachment","linq_delete_message","linq_delete_webhook_subscription","linq_edit_message","linq_get_attachment","linq_get_chat","linq_get_contact_card","linq_get_message","linq_get_webhook_subscription","linq_leave_chat","linq_list_chats","linq_list_messages","linq_list_phone_numbers","linq_list_thread","linq_list_webhook_events","linq_list_webhook_subscriptions","linq_mark_chat_read","linq_react_to_message","linq_remove_participant","linq_send_message","linq_send_voice_memo","linq_share_contact_card","linq_start_typing","linq_stop_typing","linq_update_chat","linq_update_contact_card","linq_update_webhook_subscription","llm_chat","logfire_get_token_info","logfire_get_trace","logfire_query","logfire_search_records","logrocket_create_release","logrocket_get_audit_logs","logrocket_get_highlights","logrocket_identify_user","logrocket_list_exported_sessions","logrocket_request_highlights","logs_get","logs_get_execution","logs_get_run_details","logs_query","logs_query_runs","loops_check_contact_suppression","loops_create_contact","loops_create_contact_property","loops_delete_contact","loops_find_contact","loops_get_transactional_email","loops_list_contact_properties","loops_list_mailing_lists","loops_list_transactional_emails","loops_remove_contact_suppression","loops_send_event","loops_send_transactional_email","loops_update_contact","luma_add_guests","luma_cancel_event","luma_create_event","luma_get_event","luma_get_guest","luma_get_guests","luma_list_events","luma_lookup_event","luma_send_invites","luma_update_event","luma_update_guest_status","mailchimp_add_member","mailchimp_add_member_tags","mailchimp_add_or_update_member","mailchimp_add_segment_member","mailchimp_add_subscriber_to_automation","mailchimp_archive_member","mailchimp_create_audience","mailchimp_create_batch_operation","mailchimp_create_campaign","mailchimp_create_interest","mailchimp_create_interest_category","mailchimp_create_landing_page","mailchimp_create_merge_field","mailchimp_create_segment","mailchimp_create_template","mailchimp_delete_audience","mailchimp_delete_batch_operation","mailchimp_delete_campaign","mailchimp_delete_interest","mailchimp_delete_interest_category","mailchimp_delete_landing_page","mailchimp_delete_member","mailchimp_delete_merge_field","mailchimp_delete_segment","mailchimp_delete_template","mailchimp_get_audience","mailchimp_get_audiences","mailchimp_get_automation","mailchimp_get_automations","mailchimp_get_batch_operation","mailchimp_get_batch_operations","mailchimp_get_campaign","mailchimp_get_campaign_content","mailchimp_get_campaign_report","mailchimp_get_campaign_reports","mailchimp_get_campaigns","mailchimp_get_interest","mailchimp_get_interest_categories","mailchimp_get_interest_category","mailchimp_get_interests","mailchimp_get_landing_page","mailchimp_get_landing_pages","mailchimp_get_member","mailchimp_get_member_tags","mailchimp_get_members","mailchimp_get_merge_field","mailchimp_get_merge_fields","mailchimp_get_segment","mailchimp_get_segment_members","mailchimp_get_segments","mailchimp_get_template","mailchimp_get_templates","mailchimp_pause_automation","mailchimp_publish_landing_page","mailchimp_remove_member_tags","mailchimp_remove_segment_member","mailchimp_replicate_campaign","mailchimp_schedule_campaign","mailchimp_send_campaign","mailchimp_set_campaign_content","mailchimp_start_automation","mailchimp_unarchive_member","mailchimp_unpublish_landing_page","mailchimp_unschedule_campaign","mailchimp_update_audience","mailchimp_update_campaign","mailchimp_update_interest","mailchimp_update_interest_category","mailchimp_update_landing_page","mailchimp_update_member","mailchimp_update_merge_field","mailchimp_update_segment","mailchimp_update_template","mailgun_add_list_member","mailgun_create_mailing_list","mailgun_get_domain","mailgun_get_mailing_list","mailgun_get_message","mailgun_list_domains","mailgun_list_messages","mailgun_send_message","managed_agent_archive_session","managed_agent_create_session","managed_agent_delete_session","managed_agent_get_session","managed_agent_interrupt_session","managed_agent_list_events","managed_agent_respond_custom_tool","managed_agent_respond_tool_confirmation","managed_agent_run_session","managed_agent_send_message","managed_agent_update_session","mem0_add_memories","mem0_get_memories","mem0_search_memories","memory_add","memory_delete","memory_get","memory_get_all","microsoft_ad_add_directory_role_member","microsoft_ad_add_group_member","microsoft_ad_add_user_app_role_assignment","microsoft_ad_assign_license","microsoft_ad_create_group","microsoft_ad_create_user","microsoft_ad_delete_group","microsoft_ad_delete_user","microsoft_ad_get_conditional_access_policy","microsoft_ad_get_device","microsoft_ad_get_group","microsoft_ad_get_user","microsoft_ad_list_authentication_methods","microsoft_ad_list_conditional_access_policies","microsoft_ad_list_devices","microsoft_ad_list_directory_audits","microsoft_ad_list_directory_role_members","microsoft_ad_list_directory_roles","microsoft_ad_list_group_members","microsoft_ad_list_groups","microsoft_ad_list_service_principal_app_role_assignments","microsoft_ad_list_service_principals","microsoft_ad_list_sign_ins","microsoft_ad_list_subscribed_skus","microsoft_ad_list_user_app_role_assignments","microsoft_ad_list_user_devices","microsoft_ad_list_user_licenses","microsoft_ad_list_users","microsoft_ad_remove_directory_role_member","microsoft_ad_remove_group_member","microsoft_ad_remove_user_app_role_assignment","microsoft_ad_reset_password","microsoft_ad_revoke_sign_in_sessions","microsoft_ad_set_password","microsoft_ad_update_group","microsoft_ad_update_user","microsoft_dataverse_associate","microsoft_dataverse_create_multiple","microsoft_dataverse_create_record","microsoft_dataverse_delete_record","microsoft_dataverse_disassociate","microsoft_dataverse_download_file","microsoft_dataverse_execute_action","microsoft_dataverse_execute_function","microsoft_dataverse_fetchxml_query","microsoft_dataverse_get_entity_metadata","microsoft_dataverse_get_record","microsoft_dataverse_list_records","microsoft_dataverse_search","microsoft_dataverse_update_multiple","microsoft_dataverse_update_record","microsoft_dataverse_upload_file","microsoft_dataverse_upsert_record","microsoft_dataverse_whoami","microsoft_excel_clear_range","microsoft_excel_create_table","microsoft_excel_delete_worksheet","microsoft_excel_format_range","microsoft_excel_read","microsoft_excel_read_v2","microsoft_excel_sort_range","microsoft_excel_table_add","microsoft_excel_worksheet_add","microsoft_excel_write","microsoft_excel_write_v2","microsoft_planner_create_bucket","microsoft_planner_create_plan","microsoft_planner_create_task","microsoft_planner_delete_bucket","microsoft_planner_delete_plan","microsoft_planner_delete_task","microsoft_planner_get_plan_details","microsoft_planner_get_task_details","microsoft_planner_list_buckets","microsoft_planner_list_plans","microsoft_planner_read_bucket","microsoft_planner_read_plan","microsoft_planner_read_task","microsoft_planner_update_bucket","microsoft_planner_update_plan","microsoft_planner_update_plan_details","microsoft_planner_update_task","microsoft_planner_update_task_details","microsoft_teams_delete_channel_message","microsoft_teams_delete_chat_message","microsoft_teams_get_message","microsoft_teams_list_channel_members","microsoft_teams_list_channels","microsoft_teams_list_chat_members","microsoft_teams_list_chats","microsoft_teams_list_team_members","microsoft_teams_list_teams","microsoft_teams_read_channel","microsoft_teams_read_chat","microsoft_teams_reply_to_message","microsoft_teams_set_reaction","microsoft_teams_unset_reaction","microsoft_teams_update_channel_message","microsoft_teams_update_chat_message","microsoft_teams_write_channel","microsoft_teams_write_chat","millionverifier_get_credits","millionverifier_verify_email","mintlify_create_agent_job","mintlify_create_assistant_message","mintlify_detect_ai_prose","mintlify_get_agent_job","mintlify_get_assistant_caller_stats","mintlify_get_assistant_conversations","mintlify_get_feedback","mintlify_get_feedback_by_page","mintlify_get_page_content","mintlify_get_searches","mintlify_get_update_status","mintlify_get_views","mintlify_get_visitors","mintlify_search","mintlify_send_agent_message","mintlify_trigger_automation","mintlify_trigger_preview","mintlify_trigger_update","mistral_parser","mistral_parser_v2","mistral_parser_v3","monday_archive_item","monday_change_column_value","monday_create_board","monday_create_column","monday_create_group","monday_create_item","monday_create_subitem","monday_create_update","monday_delete_item","monday_duplicate_item","monday_get_board","monday_get_groups","monday_get_item","monday_get_items","monday_list_boards","monday_move_item_to_group","monday_search_items","monday_update_item","mongodb_delete","mongodb_execute","mongodb_insert","mongodb_introspect","mongodb_query","mongodb_update","mssql_delete","mssql_execute","mssql_insert","mssql_introspect","mssql_query","mssql_update","mysql_delete","mysql_execute","mysql_insert","mysql_introspect","mysql_query","mysql_update","neo4j_create","neo4j_delete","neo4j_execute","neo4j_introspect","neo4j_merge","neo4j_query","neo4j_update","netsuite_attach_record","netsuite_batch_create_records","netsuite_batch_delete_records","netsuite_batch_get_records","netsuite_batch_update_records","netsuite_batch_upsert_records","netsuite_create_record","netsuite_delete_record","netsuite_detach_record","netsuite_execute_action","netsuite_execute_dataset","netsuite_execute_suiteql","netsuite_get_async_result","netsuite_get_async_status","netsuite_get_governance_limits","netsuite_get_record","netsuite_get_record_form","netsuite_get_record_metadata","netsuite_get_select_options","netsuite_get_server_time","netsuite_get_subresource","netsuite_list_datasets","netsuite_list_record_types","netsuite_list_records","netsuite_transform_record","netsuite_update_record","netsuite_upsert_record","neverbounce_get_credits","neverbounce_verify_email","new_relic_create_deployment_event","new_relic_get_entity","new_relic_nrql_query","new_relic_search_entities","notion_add_database_row","notion_add_database_row_v2","notion_append_blocks","notion_append_blocks_v2","notion_create_comment","notion_create_comment_v2","notion_create_database","notion_create_database_v2","notion_create_page","notion_create_page_v2","notion_delete_block","notion_delete_block_v2","notion_list_comments","notion_list_comments_v2","notion_list_users","notion_list_users_v2","notion_query_database","notion_query_database_v2","notion_read","notion_read_database","notion_read_database_v2","notion_read_v2","notion_retrieve_block","notion_retrieve_block_children","notion_retrieve_block_children_v2","notion_retrieve_block_v2","notion_retrieve_user","notion_retrieve_user_v2","notion_search","notion_search_v2","notion_update_block","notion_update_block_v2","notion_update_page","notion_update_page_v2","notion_write","notion_write_v2","obsidian_append_active","obsidian_append_note","obsidian_append_periodic_note","obsidian_create_note","obsidian_delete_note","obsidian_execute_command","obsidian_get_active","obsidian_get_note","obsidian_get_periodic_note","obsidian_list_commands","obsidian_list_files","obsidian_open_file","obsidian_patch_active","obsidian_patch_note","obsidian_search","okta_activate_group_rule","okta_activate_user","okta_add_user_to_group","okta_assign_group_to_app","okta_assign_user_role","okta_assign_user_to_app","okta_clear_user_sessions","okta_create_group","okta_create_group_rule","okta_create_user","okta_deactivate_group_rule","okta_deactivate_user","okta_delete_group","okta_delete_group_rule","okta_delete_user","okta_enroll_factor","okta_get_app","okta_get_factor","okta_get_group","okta_get_group_rule","okta_get_logs","okta_get_session","okta_get_user","okta_list_app_groups","okta_list_app_users","okta_list_apps","okta_list_factors","okta_list_group_members","okta_list_group_rules","okta_list_groups","okta_list_user_roles","okta_list_users","okta_remove_group_from_app","okta_remove_user_from_app","okta_remove_user_from_group","okta_remove_user_role","okta_reset_all_factors","okta_reset_factor","okta_reset_password","okta_revoke_session","okta_suspend_user","okta_unsuspend_user","okta_update_group","okta_update_user","onedrive_copy","onedrive_create_folder","onedrive_create_share_link","onedrive_delete","onedrive_download","onedrive_get_drive_info","onedrive_get_item","onedrive_list","onedrive_move","onedrive_search","onedrive_upload","onepassword_create_item","onepassword_delete_item","onepassword_get_item","onepassword_get_item_file","onepassword_get_vault","onepassword_list_items","onepassword_list_vaults","onepassword_replace_item","onepassword_resolve_secret","onepassword_update_item","openai_embeddings","openai_image","outlook_calendar_create_event","outlook_calendar_delete_event","outlook_calendar_get_event","outlook_calendar_list_events","outlook_calendar_respond","outlook_calendar_update_event","outlook_copy","outlook_create_folder","outlook_delete","outlook_draft","outlook_forward","outlook_get_attachment","outlook_list_attachments","outlook_list_folders","outlook_mark_read","outlook_mark_unread","outlook_move","outlook_read","outlook_reply","outlook_reply_all","outlook_search","outlook_send","outlook_update_message","pagerduty_add_note","pagerduty_create_incident","pagerduty_get_incident","pagerduty_get_service","pagerduty_list_escalation_policies","pagerduty_list_incident_alerts","pagerduty_list_incidents","pagerduty_list_oncalls","pagerduty_list_schedules","pagerduty_list_services","pagerduty_list_users","pagerduty_merge_incidents","pagerduty_send_event","pagerduty_snooze_incident","pagerduty_update_incident","parallel_deep_research","parallel_extract","parallel_search","pdl_autocomplete","pdl_bulk_company_enrich","pdl_bulk_person_enrich","pdl_clean_company","pdl_clean_location","pdl_clean_school","pdl_company_enrich","pdl_company_search","pdl_person_enrich","pdl_person_identify","pdl_person_search","perplexity_chat","perplexity_search","persona_approve_inquiry","persona_create_account","persona_create_inquiry","persona_create_report","persona_decline_inquiry","persona_expire_inquiry","persona_generate_inquiry_link","persona_get_account","persona_get_case","persona_get_document","persona_get_inquiry","persona_get_report","persona_get_verification","persona_import_accounts","persona_list_accounts","persona_list_cases","persona_list_inquiries","persona_list_inquiry_templates","persona_list_reports","persona_mark_inquiry_for_review","persona_print_inquiry_pdf","persona_redact_account","persona_redact_inquiry","persona_resume_inquiry","persona_update_account","persona_update_inquiry","pinecone_delete_vectors","pinecone_describe_index","pinecone_describe_index_stats","pinecone_fetch","pinecone_generate_embeddings","pinecone_list_indexes","pinecone_list_vector_ids","pinecone_search_text","pinecone_search_vector","pinecone_update_vector","pinecone_upsert_text","pipedrive_create_activity","pipedrive_create_deal","pipedrive_create_lead","pipedrive_create_project","pipedrive_delete_lead","pipedrive_get_activities","pipedrive_get_all_deals","pipedrive_get_deal","pipedrive_get_files","pipedrive_get_leads","pipedrive_get_mail_messages","pipedrive_get_mail_thread","pipedrive_get_pipeline_deals","pipedrive_get_pipelines","pipedrive_get_projects","pipedrive_update_activity","pipedrive_update_deal","pipedrive_update_lead","polymarket_get_activity","polymarket_get_event","polymarket_get_events","polymarket_get_holders","polymarket_get_last_trade_price","polymarket_get_leaderboard","polymarket_get_market","polymarket_get_markets","polymarket_get_midpoint","polymarket_get_orderbook","polymarket_get_positions","polymarket_get_price","polymarket_get_price_history","polymarket_get_series","polymarket_get_series_by_id","polymarket_get_spread","polymarket_get_tags","polymarket_get_tick_size","polymarket_get_trades","polymarket_search","postgresql_delete","postgresql_execute","postgresql_insert","postgresql_introspect","postgresql_query","postgresql_update","posthog_batch_events","posthog_capture_event","posthog_create_annotation","posthog_create_cohort","posthog_create_dashboard","posthog_create_experiment","posthog_create_feature_flag","posthog_create_insight","posthog_create_survey","posthog_delete_feature_flag","posthog_delete_person","posthog_delete_survey","posthog_evaluate_flags","posthog_get_cohort","posthog_get_dashboard","posthog_get_event_definition","posthog_get_experiment","posthog_get_feature_flag","posthog_get_insight","posthog_get_organization","posthog_get_person","posthog_get_project","posthog_get_property_definition","posthog_get_session_recording","posthog_get_survey","posthog_list_actions","posthog_list_annotations","posthog_list_cohorts","posthog_list_dashboards","posthog_list_event_definitions","posthog_list_experiments","posthog_list_feature_flags","posthog_list_insights","posthog_list_organizations","posthog_list_persons","posthog_list_projects","posthog_list_property_definitions","posthog_list_recording_playlists","posthog_list_session_recordings","posthog_list_surveys","posthog_query","posthog_update_cohort","posthog_update_event_definition","posthog_update_experiment","posthog_update_feature_flag","posthog_update_insight","posthog_update_property_definition","posthog_update_survey","profound_bot_logs","profound_bots_report","profound_category_assets","profound_category_personas","profound_category_prompts","profound_category_tags","profound_category_topics","profound_citation_prompts","profound_citations_report","profound_list_assets","profound_list_categories","profound_list_domains","profound_list_models","profound_list_optimizations","profound_list_personas","profound_list_regions","profound_optimization_analysis","profound_prompt_answers","profound_prompt_volume","profound_query_fanouts","profound_raw_logs","profound_referrals_report","profound_sentiment_report","profound_visibility_report","prospeo_account_information","prospeo_bulk_enrich_company","prospeo_bulk_enrich_person","prospeo_enrich_company","prospeo_enrich_person","prospeo_search_company","prospeo_search_person","prospeo_search_suggestions","pulse_parser","pulse_parser_v2","qdrant_fetch_points","qdrant_search_vector","qdrant_upsert_points","quartr_get_audio","quartr_get_company","quartr_get_event","quartr_get_event_summary","quartr_get_report","quartr_get_slide_deck","quartr_get_transcript","quartr_list_audio","quartr_list_companies","quartr_list_document_types","quartr_list_documents","quartr_list_event_types","quartr_list_events","quartr_list_live_events","quartr_list_reports","quartr_list_slide_decks","quartr_list_transcripts","quiver_image_to_svg","quiver_list_models","quiver_text_to_svg","rabbitmq_create_binding","rabbitmq_create_exchange","rabbitmq_create_policy","rabbitmq_create_queue","rabbitmq_delete_binding","rabbitmq_delete_exchange","rabbitmq_delete_policy","rabbitmq_delete_queue","rabbitmq_get_exchange","rabbitmq_get_messages","rabbitmq_get_overview","rabbitmq_get_queue","rabbitmq_health_check","rabbitmq_list_bindings","rabbitmq_list_channels","rabbitmq_list_connections","rabbitmq_list_consumers","rabbitmq_list_exchange_bindings","rabbitmq_list_exchanges","rabbitmq_list_nodes","rabbitmq_list_policies","rabbitmq_list_queues","rabbitmq_list_vhosts","rabbitmq_publish_message","rabbitmq_purge_queue","railway_create_environment","railway_create_project","railway_create_service","railway_delete_environment","railway_delete_project","railway_delete_service","railway_delete_variable","railway_deploy_service","railway_get_deployment","railway_get_deployment_logs","railway_get_project","railway_list_deployments","railway_list_project_members","railway_list_projects","railway_list_variables","railway_restart_deployment","railway_rollback_deployment","railway_transfer_project","railway_update_project","railway_upsert_variable","rb2b_credit_check","rb2b_email_to_activity","rb2b_hem_to_best_linkedin","rb2b_hem_to_business_profile","rb2b_hem_to_linkedin","rb2b_hem_to_maid","rb2b_ip_to_company","rb2b_ip_to_hem","rb2b_ip_to_maid","rb2b_linkedin_slug_search","rb2b_linkedin_to_best_personal_email","rb2b_linkedin_to_business_profile","rb2b_linkedin_to_hashed_emails","rb2b_linkedin_to_mobile_phone","rb2b_linkedin_to_personal_email","rds_delete","rds_execute","rds_insert","rds_introspect","rds_query","rds_update","reddit_delete","reddit_edit","reddit_get_comments","reddit_get_controversial","reddit_get_info","reddit_get_me","reddit_get_messages","reddit_get_posts","reddit_get_saved","reddit_get_subreddit_info","reddit_get_subreddit_rules","reddit_get_user","reddit_get_user_comments","reddit_get_user_posts","reddit_hide","reddit_hot_posts","reddit_list_my_subreddits","reddit_lock","reddit_mark_all_read","reddit_mark_read","reddit_marknsfw","reddit_mod_approve","reddit_mod_distinguish","reddit_mod_remove","reddit_mod_sticky","reddit_reply","reddit_report","reddit_save","reddit_search","reddit_search_subreddits","reddit_send_message","reddit_submit_post","reddit_subscribe","reddit_unhide","reddit_unlock","reddit_unmarknsfw","reddit_unsave","reddit_vote","redis_command","redis_delete","redis_exists","redis_expire","redis_get","redis_hdel","redis_hget","redis_hgetall","redis_hset","redis_incr","redis_incrby","redis_keys","redis_llen","redis_lpop","redis_lpush","redis_lrange","redis_persist","redis_rpop","redis_rpush","redis_set","redis_setnx","redis_ttl","reducto_parser","reducto_parser_v2","resend_cancel_email","resend_create_audience","resend_create_broadcast","resend_create_contact","resend_delete_audience","resend_delete_contact","resend_get_audience","resend_get_broadcast","resend_get_contact","resend_get_email","resend_list_audiences","resend_list_contacts","resend_list_domains","resend_send","resend_send_broadcast","resend_update_contact","revenuecat_create_purchase","revenuecat_defer_google_subscription","revenuecat_delete_customer","revenuecat_get_customer","revenuecat_grant_entitlement","revenuecat_list_offerings","revenuecat_refund_google_subscription","revenuecat_revoke_entitlement","revenuecat_revoke_google_subscription","revenuecat_update_subscriber_attributes","rippling_bulk_create_custom_object_records","rippling_bulk_delete_custom_object_records","rippling_bulk_update_custom_object_records","rippling_create_business_partner","rippling_create_business_partner_group","rippling_create_custom_app","rippling_create_custom_object","rippling_create_custom_object_field","rippling_create_custom_object_record","rippling_create_custom_page","rippling_create_custom_setting","rippling_create_department","rippling_create_draft_hires","rippling_create_object_category","rippling_create_title","rippling_create_work_location","rippling_delete_business_partner","rippling_delete_business_partner_group","rippling_delete_custom_app","rippling_delete_custom_object","rippling_delete_custom_object_field","rippling_delete_custom_object_record","rippling_delete_custom_page","rippling_delete_custom_setting","rippling_delete_object_category","rippling_delete_title","rippling_delete_work_location","rippling_get_business_partner","rippling_get_business_partner_group","rippling_get_current_user","rippling_get_custom_app","rippling_get_custom_object","rippling_get_custom_object_field","rippling_get_custom_object_record","rippling_get_custom_object_record_by_external_id","rippling_get_custom_page","rippling_get_custom_setting","rippling_get_department","rippling_get_employment_type","rippling_get_job_function","rippling_get_object_category","rippling_get_report_run","rippling_get_supergroup","rippling_get_team","rippling_get_title","rippling_get_user","rippling_get_work_location","rippling_get_worker","rippling_list_business_partner_groups","rippling_list_business_partners","rippling_list_companies","rippling_list_custom_apps","rippling_list_custom_fields","rippling_list_custom_object_fields","rippling_list_custom_object_records","rippling_list_custom_objects","rippling_list_custom_pages","rippling_list_custom_settings","rippling_list_departments","rippling_list_employment_types","rippling_list_entitlements","rippling_list_job_functions","rippling_list_object_categories","rippling_list_supergroup_exclusion_members","rippling_list_supergroup_inclusion_members","rippling_list_supergroup_members","rippling_list_supergroups","rippling_list_teams","rippling_list_titles","rippling_list_users","rippling_list_work_locations","rippling_list_workers","rippling_query_custom_object_records","rippling_trigger_report_run","rippling_update_custom_app","rippling_update_custom_object","rippling_update_custom_object_field","rippling_update_custom_object_record","rippling_update_custom_page","rippling_update_custom_setting","rippling_update_department","rippling_update_object_category","rippling_update_supergroup_exclusion_members","rippling_update_supergroup_inclusion_members","rippling_update_title","rippling_update_work_location","rocketlane_add_field_option","rocketlane_add_project_members","rocketlane_add_task_assignees","rocketlane_add_task_dependencies","rocketlane_add_task_followers","rocketlane_archive_project","rocketlane_assign_placeholders","rocketlane_create_field","rocketlane_create_phase","rocketlane_create_project","rocketlane_create_space","rocketlane_create_space_document","rocketlane_create_task","rocketlane_create_time_entry","rocketlane_create_time_off","rocketlane_delete_field","rocketlane_delete_phase","rocketlane_delete_project","rocketlane_delete_space","rocketlane_delete_space_document","rocketlane_delete_task","rocketlane_delete_time_entry","rocketlane_delete_time_off","rocketlane_get_field","rocketlane_get_invoice","rocketlane_get_invoice_line_items","rocketlane_get_invoice_payments","rocketlane_get_phase","rocketlane_get_project","rocketlane_get_space","rocketlane_get_space_document","rocketlane_get_task","rocketlane_get_time_entry","rocketlane_get_time_off","rocketlane_get_user","rocketlane_import_template","rocketlane_list_fields","rocketlane_list_invoices","rocketlane_list_phases","rocketlane_list_placeholders","rocketlane_list_projects","rocketlane_list_resource_allocations","rocketlane_list_space_documents","rocketlane_list_spaces","rocketlane_list_tasks","rocketlane_list_time_entries","rocketlane_list_time_entry_categories","rocketlane_list_time_offs","rocketlane_list_users","rocketlane_move_task_to_phase","rocketlane_remove_project_members","rocketlane_remove_task_assignees","rocketlane_remove_task_dependencies","rocketlane_remove_task_followers","rocketlane_search_time_entries","rocketlane_unassign_placeholders","rocketlane_update_field","rocketlane_update_field_option","rocketlane_update_phase","rocketlane_update_project","rocketlane_update_space","rocketlane_update_space_document","rocketlane_update_task","rocketlane_update_time_entry","rootly_acknowledge_alert","rootly_add_incident_event","rootly_add_subscribers","rootly_assign_incident_role","rootly_create_action_item","rootly_create_alert","rootly_create_incident","rootly_create_status_page_event","rootly_delete_action_item","rootly_delete_incident","rootly_escalate_alert","rootly_get_alert","rootly_get_incident","rootly_list_action_items","rootly_list_alerts","rootly_list_causes","rootly_list_environments","rootly_list_escalation_policies","rootly_list_functionalities","rootly_list_incident_events","rootly_list_incident_roles","rootly_list_incident_types","rootly_list_incidents","rootly_list_on_calls","rootly_list_playbooks","rootly_list_retrospectives","rootly_list_schedules","rootly_list_services","rootly_list_severities","rootly_list_teams","rootly_list_users","rootly_mitigate_incident","rootly_remove_subscribers","rootly_resolve_alert","rootly_resolve_incident","rootly_run_workflow","rootly_snooze_alert","rootly_unassign_incident_role","rootly_update_action_item","rootly_update_alert","rootly_update_incident","s3_copy_object","s3_create_bucket","s3_delete_bucket","s3_delete_object","s3_delete_objects","s3_get_object","s3_head_object","s3_list_buckets","s3_list_objects","s3_presigned_url","s3_put_object","salesforce_create_account","salesforce_create_case","salesforce_create_contact","salesforce_create_custom_field","salesforce_create_custom_object","salesforce_create_lead","salesforce_create_opportunity","salesforce_create_task","salesforce_delete_account","salesforce_delete_case","salesforce_delete_contact","salesforce_delete_custom_field","salesforce_delete_lead","salesforce_delete_opportunity","salesforce_delete_task","salesforce_describe_object","salesforce_get_accounts","salesforce_get_cases","salesforce_get_contacts","salesforce_get_dashboard","salesforce_get_leads","salesforce_get_opportunities","salesforce_get_report","salesforce_get_tasks","salesforce_list_dashboards","salesforce_list_objects","salesforce_list_report_types","salesforce_list_reports","salesforce_query","salesforce_query_more","salesforce_refresh_dashboard","salesforce_run_report","salesforce_tooling_query","salesforce_update_account","salesforce_update_case","salesforce_update_contact","salesforce_update_custom_field","salesforce_update_lead","salesforce_update_opportunity","salesforce_update_task","sap_concur_approve_expense_report","sap_concur_associate_attendees","sap_concur_create_cash_advance","sap_concur_create_expected_expense","sap_concur_create_expense_report","sap_concur_create_list_item","sap_concur_create_purchase_request","sap_concur_create_quick_expense","sap_concur_create_quick_expense_with_image","sap_concur_create_report_comment","sap_concur_create_travel_request","sap_concur_create_user","sap_concur_delete_expected_expense","sap_concur_delete_expense","sap_concur_delete_expense_report","sap_concur_delete_list_item","sap_concur_delete_travel_request","sap_concur_delete_user","sap_concur_get_allocation","sap_concur_get_budget","sap_concur_get_cash_advance","sap_concur_get_expected_expense","sap_concur_get_expense","sap_concur_get_expense_report","sap_concur_get_itemizations","sap_concur_get_itinerary","sap_concur_get_list","sap_concur_get_list_item","sap_concur_get_purchase_request","sap_concur_get_receipt","sap_concur_get_receipt_status","sap_concur_get_request_cash_advance","sap_concur_get_travel_profile","sap_concur_get_travel_request","sap_concur_get_user","sap_concur_issue_cash_advance","sap_concur_list_allocations","sap_concur_list_attendee_associations","sap_concur_list_budget_categories","sap_concur_list_budgets","sap_concur_list_exceptions","sap_concur_list_expected_expenses","sap_concur_list_expense_reports","sap_concur_list_expenses","sap_concur_list_itineraries","sap_concur_list_list_items","sap_concur_list_lists","sap_concur_list_receipts","sap_concur_list_report_comments","sap_concur_list_reports_to_approve","sap_concur_list_travel_profiles_summary","sap_concur_list_travel_request_comments","sap_concur_list_travel_requests","sap_concur_list_users","sap_concur_move_travel_request","sap_concur_recall_expense_report","sap_concur_remove_all_attendees","sap_concur_search_locations","sap_concur_search_users","sap_concur_send_back_expense_report","sap_concur_submit_expense_report","sap_concur_update_allocation","sap_concur_update_expected_expense","sap_concur_update_expense","sap_concur_update_expense_report","sap_concur_update_list_item","sap_concur_update_travel_request","sap_concur_update_user","sap_concur_upload_exchange_rates","sap_concur_upload_receipt_image","sap_s4hana_create_business_partner","sap_s4hana_create_purchase_order","sap_s4hana_create_purchase_requisition","sap_s4hana_create_sales_order","sap_s4hana_delete_sales_order","sap_s4hana_get_billing_document","sap_s4hana_get_business_partner","sap_s4hana_get_customer","sap_s4hana_get_inbound_delivery","sap_s4hana_get_material_document","sap_s4hana_get_outbound_delivery","sap_s4hana_get_product","sap_s4hana_get_purchase_order","sap_s4hana_get_purchase_requisition","sap_s4hana_get_sales_order","sap_s4hana_get_supplier","sap_s4hana_get_supplier_invoice","sap_s4hana_list_billing_documents","sap_s4hana_list_business_partners","sap_s4hana_list_customers","sap_s4hana_list_inbound_deliveries","sap_s4hana_list_material_documents","sap_s4hana_list_material_stock","sap_s4hana_list_outbound_deliveries","sap_s4hana_list_products","sap_s4hana_list_purchase_orders","sap_s4hana_list_purchase_requisitions","sap_s4hana_list_sales_orders","sap_s4hana_list_supplier_invoices","sap_s4hana_list_suppliers","sap_s4hana_odata_query","sap_s4hana_update_business_partner","sap_s4hana_update_customer","sap_s4hana_update_product","sap_s4hana_update_purchase_order","sap_s4hana_update_purchase_requisition","sap_s4hana_update_sales_order","sap_s4hana_update_supplier","search_tool","secrets_manager_create_secret","secrets_manager_delete_secret","secrets_manager_describe_secret","secrets_manager_get_secret","secrets_manager_list_secrets","secrets_manager_restore_secret","secrets_manager_rotate_secret","secrets_manager_tag_resource","secrets_manager_untag_resource","secrets_manager_update_secret","sendblue_evaluate_service","sendblue_get_message","sendblue_send_group_message","sendblue_send_message","sendblue_send_typing_indicator","sendgrid_add_contact","sendgrid_add_contacts_to_list","sendgrid_create_list","sendgrid_create_template","sendgrid_create_template_version","sendgrid_delete_contacts","sendgrid_delete_list","sendgrid_delete_template","sendgrid_get_contact","sendgrid_get_list","sendgrid_get_template","sendgrid_list_all_lists","sendgrid_list_templates","sendgrid_remove_contacts_from_list","sendgrid_search_contacts","sendgrid_send_mail","sentry_events_get","sentry_events_list","sentry_issues_get","sentry_issues_list","sentry_issues_update","sentry_projects_create","sentry_projects_get","sentry_projects_list","sentry_projects_update","sentry_releases_create","sentry_releases_deploy","sentry_releases_list","sentry_teams_list","serper_search","servicenow_add_incident_comment","servicenow_aggregate","servicenow_close_incident","servicenow_create_change_request","servicenow_create_incident","servicenow_create_record","servicenow_delete_record","servicenow_download_attachment","servicenow_find_user","servicenow_get_change_next_states","servicenow_get_change_request","servicenow_get_ci","servicenow_get_incident","servicenow_get_knowledge_article","servicenow_get_requested_item","servicenow_list_approvals","servicenow_list_attachments","servicenow_list_catalog_items","servicenow_list_change_requests","servicenow_list_change_tasks","servicenow_list_ci_relationships","servicenow_list_group_members","servicenow_list_incidents","servicenow_list_requested_items","servicenow_order_catalog_item","servicenow_read_record","servicenow_resolve_incident","servicenow_search_cis","servicenow_search_knowledge","servicenow_update_approval","servicenow_update_change_request","servicenow_update_change_state","servicenow_update_incident","servicenow_update_record","servicenow_upload_attachment","ses_create_configuration_set","ses_create_email_identity","ses_create_template","ses_delete_email_identity","ses_delete_suppressed_destination","ses_delete_template","ses_get_account","ses_get_email_identity","ses_get_suppressed_destination","ses_get_template","ses_list_identities","ses_list_suppressed_destinations","ses_list_templates","ses_put_suppressed_destination","ses_send_bulk_email","ses_send_custom_verification_email","ses_send_email","ses_send_templated_email","ses_update_template","sftp_delete","sftp_download","sftp_list","sftp_mkdir","sftp_upload","sharepoint_add_list_items","sharepoint_create_list","sharepoint_create_page","sharepoint_delete_file","sharepoint_delete_list_item","sharepoint_delete_page","sharepoint_download_file","sharepoint_get_drive_item","sharepoint_get_list","sharepoint_get_list_item","sharepoint_list_sites","sharepoint_publish_page","sharepoint_read_page","sharepoint_update_list","sharepoint_update_page","sharepoint_upload_file","shopify_adjust_inventory","shopify_cancel_order","shopify_create_customer","shopify_create_fulfillment","shopify_create_product","shopify_delete_customer","shopify_delete_product","shopify_get_collection","shopify_get_customer","shopify_get_inventory_level","shopify_get_order","shopify_get_product","shopify_list_collections","shopify_list_customers","shopify_list_inventory_items","shopify_list_locations","shopify_list_orders","shopify_list_products","shopify_update_customer","shopify_update_order","shopify_update_product","similarweb_bounce_rate","similarweb_page_views","similarweb_pages_per_visit","similarweb_traffic_visits","similarweb_visit_duration","similarweb_website_overview","sixtyfour_enrich_company","sixtyfour_enrich_lead","sixtyfour_find_email","sixtyfour_find_phone","slack_add_reaction","slack_archive_conversation","slack_canvas","slack_create_channel_canvas","slack_create_conversation","slack_delete_canvas","slack_delete_message","slack_delete_scheduled_message","slack_download","slack_edit_canvas","slack_ephemeral_message","slack_get_canvas","slack_get_channel_history","slack_get_channel_info","slack_get_message","slack_get_permalink","slack_get_thread","slack_get_thread_replies","slack_get_user","slack_get_user_presence","slack_invite_to_conversation","slack_list_canvases","slack_list_channels","slack_list_members","slack_list_scheduled_messages","slack_list_users","slack_lookup_canvas_sections","slack_message","slack_message_reader","slack_open_view","slack_publish_view","slack_push_view","slack_remove_reaction","slack_rename_conversation","slack_schedule_message","slack_set_conversation_purpose","slack_set_conversation_topic","slack_set_status","slack_set_suggested_prompts","slack_set_title","slack_update_message","slack_update_view","smartlead_add_email_accounts_to_campaign","smartlead_add_leads_to_campaign","smartlead_create_campaign","smartlead_create_lead_list","smartlead_delete_campaign","smartlead_delete_campaign_webhook","smartlead_delete_lead_from_campaign","smartlead_delete_lead_list","smartlead_duplicate_campaign","smartlead_export_campaign_leads","smartlead_get_campaign","smartlead_get_campaign_analytics","smartlead_get_campaign_analytics_by_date","smartlead_get_campaign_lead_statistics","smartlead_get_campaign_mailbox_statistics","smartlead_get_campaign_sequences","smartlead_get_campaign_statistics","smartlead_get_campaign_top_level_analytics_by_date","smartlead_get_campaign_webhook_summary","smartlead_get_lead_by_email","smartlead_get_lead_by_id","smartlead_get_lead_list","smartlead_get_lead_message_history","smartlead_list_campaign_email_accounts","smartlead_list_campaign_leads","smartlead_list_campaign_webhooks","smartlead_list_campaigns","smartlead_list_clients","smartlead_list_email_accounts","smartlead_list_inbox_replies","smartlead_list_lead_activities","smartlead_list_lead_categories","smartlead_list_lead_lists","smartlead_mark_lead_complete","smartlead_pause_lead","smartlead_remove_email_accounts_from_campaign","smartlead_resume_lead","smartlead_save_campaign_sequences","smartlead_unsubscribe_lead_from_campaign","smartlead_unsubscribe_lead_globally","smartlead_update_campaign_schedule","smartlead_update_campaign_settings","smartlead_update_campaign_status","smartlead_update_lead","smartlead_update_lead_category","smartlead_update_lead_list","smartlead_upsert_campaign_webhook","sms_send","smtp_send_mail","snowflake_alter_warehouse","snowflake_call_procedure","snowflake_cancel_statement","snowflake_cancel_task_run","snowflake_delete_rows","snowflake_execute_sql","snowflake_get_statement","snowflake_get_task","snowflake_get_task_run","snowflake_get_task_run_output","snowflake_get_warehouse","snowflake_insert_rows","snowflake_introspect_schema","snowflake_list_copy_history","snowflake_list_databases","snowflake_list_query_history","snowflake_list_schemas","snowflake_list_tables","snowflake_list_task_runs","snowflake_list_tasks","snowflake_list_warehouses","snowflake_load_data","snowflake_resume_task","snowflake_resume_warehouse","snowflake_run_task","snowflake_suspend_task","snowflake_suspend_warehouse","snowflake_unload_data","snowflake_update_rows","snowflake_upsert_rows","splunk_cancel_search_job","splunk_create_search_job","splunk_dispatch_saved_search","splunk_get_fired_alerts","splunk_get_saved_search","splunk_get_search_job","splunk_get_search_results","splunk_list_apps","splunk_list_fired_alerts","splunk_list_indexes","splunk_list_saved_searches","splunk_run_search","sportmonks_core_get_cities","sportmonks_core_get_city","sportmonks_core_get_continent","sportmonks_core_get_continents","sportmonks_core_get_countries","sportmonks_core_get_country","sportmonks_core_get_entity_filters","sportmonks_core_get_my_usage","sportmonks_core_get_region","sportmonks_core_get_regions","sportmonks_core_get_timezones","sportmonks_core_get_type","sportmonks_core_get_type_by_entity","sportmonks_core_get_types","sportmonks_core_search_cities","sportmonks_core_search_countries","sportmonks_core_search_regions","sportmonks_football_expected_by_player","sportmonks_football_expected_by_team","sportmonks_football_get_all_commentaries","sportmonks_football_get_all_fixtures","sportmonks_football_get_all_players","sportmonks_football_get_all_rivals","sportmonks_football_get_all_teams","sportmonks_football_get_all_transfer_rumours","sportmonks_football_get_all_transfers","sportmonks_football_get_brackets_by_season","sportmonks_football_get_coach","sportmonks_football_get_coaches","sportmonks_football_get_coaches_by_country","sportmonks_football_get_commentaries_by_fixture","sportmonks_football_get_current_leagues_by_team","sportmonks_football_get_expected_lineups_by_player","sportmonks_football_get_expected_lineups_by_team","sportmonks_football_get_extended_team_squad","sportmonks_football_get_fixture","sportmonks_football_get_fixtures_by_date","sportmonks_football_get_fixtures_by_date_range","sportmonks_football_get_fixtures_by_date_range_for_team","sportmonks_football_get_fixtures_by_ids","sportmonks_football_get_grouped_standings_by_round","sportmonks_football_get_head_to_head","sportmonks_football_get_inplay_livescores","sportmonks_football_get_latest_coaches","sportmonks_football_get_latest_fixtures","sportmonks_football_get_latest_livescores","sportmonks_football_get_latest_players","sportmonks_football_get_latest_totw","sportmonks_football_get_latest_transfers","sportmonks_football_get_league","sportmonks_football_get_leagues","sportmonks_football_get_leagues_by_country","sportmonks_football_get_leagues_by_date","sportmonks_football_get_leagues_by_team","sportmonks_football_get_live_leagues","sportmonks_football_get_live_probabilities","sportmonks_football_get_live_probabilities_by_fixture","sportmonks_football_get_live_standings_by_league","sportmonks_football_get_livescores","sportmonks_football_get_match_facts","sportmonks_football_get_match_facts_by_date_range","sportmonks_football_get_match_facts_by_fixture","sportmonks_football_get_match_facts_by_league","sportmonks_football_get_past_fixtures_by_tv_station","sportmonks_football_get_player","sportmonks_football_get_players_by_country","sportmonks_football_get_postmatch_news","sportmonks_football_get_postmatch_news_by_season","sportmonks_football_get_predictability_by_league","sportmonks_football_get_prematch_news","sportmonks_football_get_prematch_news_by_season","sportmonks_football_get_prematch_news_upcoming","sportmonks_football_get_probabilities","sportmonks_football_get_probabilities_by_fixture","sportmonks_football_get_referee","sportmonks_football_get_referees","sportmonks_football_get_referees_by_country","sportmonks_football_get_referees_by_season","sportmonks_football_get_rivals_by_team","sportmonks_football_get_round","sportmonks_football_get_round_statistics","sportmonks_football_get_rounds","sportmonks_football_get_rounds_by_season","sportmonks_football_get_schedules_by_season","sportmonks_football_get_schedules_by_season_and_team","sportmonks_football_get_schedules_by_team","sportmonks_football_get_season","sportmonks_football_get_seasons","sportmonks_football_get_seasons_by_team","sportmonks_football_get_stage","sportmonks_football_get_stage_statistics","sportmonks_football_get_stages","sportmonks_football_get_stages_by_season","sportmonks_football_get_standing_corrections_by_season","sportmonks_football_get_standings","sportmonks_football_get_standings_by_round","sportmonks_football_get_standings_by_season","sportmonks_football_get_state","sportmonks_football_get_states","sportmonks_football_get_team","sportmonks_football_get_team_rankings","sportmonks_football_get_team_rankings_by_date","sportmonks_football_get_team_rankings_by_team","sportmonks_football_get_team_squad","sportmonks_football_get_team_squad_by_season","sportmonks_football_get_teams_by_country","sportmonks_football_get_teams_by_season","sportmonks_football_get_topscorers_by_season","sportmonks_football_get_topscorers_by_stage","sportmonks_football_get_totw","sportmonks_football_get_totw_by_round","sportmonks_football_get_transfer","sportmonks_football_get_transfer_rumour","sportmonks_football_get_transfer_rumours_between_dates","sportmonks_football_get_transfer_rumours_by_player","sportmonks_football_get_transfer_rumours_by_team","sportmonks_football_get_transfers_between_dates","sportmonks_football_get_transfers_by_player","sportmonks_football_get_transfers_by_team","sportmonks_football_get_tv_station","sportmonks_football_get_tv_stations","sportmonks_football_get_tv_stations_by_fixture","sportmonks_football_get_upcoming_fixtures_by_market","sportmonks_football_get_upcoming_fixtures_by_tv_station","sportmonks_football_get_value_bets","sportmonks_football_get_value_bets_by_fixture","sportmonks_football_get_venue","sportmonks_football_get_venues","sportmonks_football_get_venues_by_season","sportmonks_football_search_coaches","sportmonks_football_search_fixtures","sportmonks_football_search_leagues","sportmonks_football_search_players","sportmonks_football_search_referees","sportmonks_football_search_rounds","sportmonks_football_search_seasons","sportmonks_football_search_stages","sportmonks_football_search_teams","sportmonks_football_search_venues","sportmonks_motorsport_get_all_fixtures","sportmonks_motorsport_get_current_leagues_by_team","sportmonks_motorsport_get_driver","sportmonks_motorsport_get_driver_standings","sportmonks_motorsport_get_driver_standings_by_season","sportmonks_motorsport_get_drivers","sportmonks_motorsport_get_drivers_by_country","sportmonks_motorsport_get_drivers_by_season","sportmonks_motorsport_get_fixture","sportmonks_motorsport_get_fixtures_by_date","sportmonks_motorsport_get_fixtures_by_date_range","sportmonks_motorsport_get_fixtures_by_ids","sportmonks_motorsport_get_laps_by_fixture","sportmonks_motorsport_get_laps_by_fixture_and_driver","sportmonks_motorsport_get_laps_by_fixture_and_lap","sportmonks_motorsport_get_latest_laps_by_fixture","sportmonks_motorsport_get_latest_pitstops_by_fixture","sportmonks_motorsport_get_latest_stints_by_fixture","sportmonks_motorsport_get_latest_updated_drivers","sportmonks_motorsport_get_latest_updated_fixtures","sportmonks_motorsport_get_league","sportmonks_motorsport_get_leagues","sportmonks_motorsport_get_leagues_by_country","sportmonks_motorsport_get_leagues_by_date","sportmonks_motorsport_get_leagues_by_live","sportmonks_motorsport_get_leagues_by_team","sportmonks_motorsport_get_livescores","sportmonks_motorsport_get_pitstops_by_fixture","sportmonks_motorsport_get_pitstops_by_fixture_and_driver","sportmonks_motorsport_get_pitstops_by_fixture_and_lap","sportmonks_motorsport_get_race_results_by_season_and_driver","sportmonks_motorsport_get_race_results_by_season_and_team","sportmonks_motorsport_get_schedules_by_season","sportmonks_motorsport_get_season","sportmonks_motorsport_get_seasons","sportmonks_motorsport_get_stage","sportmonks_motorsport_get_stages","sportmonks_motorsport_get_stages_by_season","sportmonks_motorsport_get_state","sportmonks_motorsport_get_states","sportmonks_motorsport_get_stints_by_fixture","sportmonks_motorsport_get_stints_by_fixture_and_driver","sportmonks_motorsport_get_stints_by_fixture_and_stint","sportmonks_motorsport_get_team","sportmonks_motorsport_get_team_standings","sportmonks_motorsport_get_team_standings_by_season","sportmonks_motorsport_get_teams","sportmonks_motorsport_get_teams_by_country","sportmonks_motorsport_get_teams_by_season","sportmonks_motorsport_get_venue","sportmonks_motorsport_get_venues","sportmonks_motorsport_get_venues_by_season","sportmonks_motorsport_search_drivers","sportmonks_motorsport_search_leagues","sportmonks_motorsport_search_stages","sportmonks_motorsport_search_teams","sportmonks_motorsport_search_venues","sportmonks_odds_get_all_historical_odds","sportmonks_odds_get_all_inplay_odds","sportmonks_odds_get_all_pre_match_odds","sportmonks_odds_get_all_premium_odds","sportmonks_odds_get_bookmaker","sportmonks_odds_get_bookmaker_event_ids_by_fixture","sportmonks_odds_get_bookmakers","sportmonks_odds_get_bookmakers_by_fixture","sportmonks_odds_get_inplay_odds_by_fixture","sportmonks_odds_get_inplay_odds_by_fixture_and_bookmaker","sportmonks_odds_get_inplay_odds_by_fixture_and_market","sportmonks_odds_get_last_updated_inplay_odds","sportmonks_odds_get_last_updated_pre_match_odds","sportmonks_odds_get_market","sportmonks_odds_get_markets","sportmonks_odds_get_pre_match_odds_by_fixture","sportmonks_odds_get_pre_match_odds_by_fixture_and_bookmaker","sportmonks_odds_get_pre_match_odds_by_fixture_and_market","sportmonks_odds_get_premium_odds_by_fixture","sportmonks_odds_get_premium_odds_by_fixture_and_bookmaker","sportmonks_odds_get_premium_odds_by_fixture_and_market","sportmonks_odds_get_updated_historical_odds_between","sportmonks_odds_get_updated_premium_odds_between","sportmonks_odds_search_bookmakers","sportmonks_odds_search_markets","spotify_add_playlist_cover","spotify_add_to_queue","spotify_add_tracks_to_playlist","spotify_check_following","spotify_check_playlist_followers","spotify_check_saved_albums","spotify_check_saved_audiobooks","spotify_check_saved_episodes","spotify_check_saved_shows","spotify_check_saved_tracks","spotify_create_playlist","spotify_follow_artists","spotify_follow_playlist","spotify_get_album","spotify_get_album_tracks","spotify_get_albums","spotify_get_artist","spotify_get_artist_albums","spotify_get_artist_top_tracks","spotify_get_artists","spotify_get_audiobook","spotify_get_audiobook_chapters","spotify_get_audiobooks","spotify_get_categories","spotify_get_current_user","spotify_get_currently_playing","spotify_get_devices","spotify_get_episode","spotify_get_episodes","spotify_get_followed_artists","spotify_get_markets","spotify_get_new_releases","spotify_get_playback_state","spotify_get_playlist","spotify_get_playlist_cover","spotify_get_playlist_tracks","spotify_get_queue","spotify_get_recently_played","spotify_get_saved_albums","spotify_get_saved_audiobooks","spotify_get_saved_episodes","spotify_get_saved_shows","spotify_get_saved_tracks","spotify_get_show","spotify_get_show_episodes","spotify_get_shows","spotify_get_top_artists","spotify_get_top_tracks","spotify_get_track","spotify_get_tracks","spotify_get_user_playlists","spotify_get_user_profile","spotify_pause","spotify_play","spotify_remove_saved_albums","spotify_remove_saved_audiobooks","spotify_remove_saved_episodes","spotify_remove_saved_shows","spotify_remove_saved_tracks","spotify_remove_tracks_from_playlist","spotify_reorder_playlist_items","spotify_replace_playlist_items","spotify_save_albums","spotify_save_audiobooks","spotify_save_episodes","spotify_save_shows","spotify_save_tracks","spotify_search","spotify_seek","spotify_set_repeat","spotify_set_shuffle","spotify_set_volume","spotify_skip_next","spotify_skip_previous","spotify_transfer_playback","spotify_unfollow_artists","spotify_unfollow_playlist","spotify_update_playlist","sqs_send","square_batch_retrieve_inventory_counts","square_cancel_invoice","square_cancel_payment","square_complete_payment","square_create_catalog_image","square_create_customer","square_create_invoice","square_create_order","square_create_payment","square_delete_catalog_object","square_delete_customer","square_delete_invoice","square_get_catalog_object","square_get_customer","square_get_invoice","square_get_location","square_get_order","square_get_payment","square_get_refund","square_list_catalog","square_list_customers","square_list_invoices","square_list_locations","square_list_payments","square_list_refunds","square_pay_order","square_publish_invoice","square_refund_payment","square_search_catalog_objects","square_search_customers","square_search_invoices","square_search_orders","square_update_customer","square_upsert_catalog_object","ssh_check_command_exists","ssh_check_file_exists","ssh_create_directory","ssh_delete_file","ssh_download_file","ssh_execute_command","ssh_execute_script","ssh_get_system_info","ssh_list_directory","ssh_move_rename","ssh_read_file_content","ssh_upload_file","ssh_write_file_content","stagehand_agent","stagehand_extract","stripe_cancel_payment_intent","stripe_cancel_subscription","stripe_capture_charge","stripe_capture_payment_intent","stripe_confirm_payment_intent","stripe_create_charge","stripe_create_customer","stripe_create_invoice","stripe_create_payment_intent","stripe_create_price","stripe_create_product","stripe_create_subscription","stripe_delete_customer","stripe_delete_invoice","stripe_delete_product","stripe_finalize_invoice","stripe_list_charges","stripe_list_customers","stripe_list_events","stripe_list_invoices","stripe_list_payment_intents","stripe_list_prices","stripe_list_products","stripe_list_subscriptions","stripe_pay_invoice","stripe_resume_subscription","stripe_retrieve_charge","stripe_retrieve_customer","stripe_retrieve_event","stripe_retrieve_invoice","stripe_retrieve_payment_intent","stripe_retrieve_price","stripe_retrieve_product","stripe_retrieve_subscription","stripe_search_charges","stripe_search_customers","stripe_search_invoices","stripe_search_payment_intents","stripe_search_prices","stripe_search_products","stripe_search_subscriptions","stripe_send_invoice","stripe_update_charge","stripe_update_customer","stripe_update_invoice","stripe_update_payment_intent","stripe_update_price","stripe_update_product","stripe_update_subscription","stripe_void_invoice","sts_assume_role","sts_assume_role_with_saml","sts_assume_role_with_web_identity","sts_get_access_key_info","sts_get_caller_identity","sts_get_session_token","stt_assemblyai","stt_assemblyai_v2","stt_deepgram","stt_deepgram_v2","stt_elevenlabs","stt_elevenlabs_v2","stt_gemini","stt_gemini_v2","stt_whisper","stt_whisper_v2","supabase_count","supabase_delete","supabase_get_row","supabase_insert","supabase_introspect","supabase_invoke_function","supabase_query","supabase_rpc","supabase_storage_copy","supabase_storage_create_bucket","supabase_storage_create_signed_upload_url","supabase_storage_create_signed_url","supabase_storage_delete","supabase_storage_delete_bucket","supabase_storage_download","supabase_storage_empty_bucket","supabase_storage_get_public_url","supabase_storage_list","supabase_storage_list_buckets","supabase_storage_move","supabase_storage_update_bucket","supabase_storage_upload","supabase_text_search","supabase_update","supabase_upsert","supabase_vector_search","table_batch_insert_rows","table_create","table_delete_row","table_delete_rows_by_filter","table_get_row","table_get_schema","table_insert_row","table_list","table_query_rows","table_query_rows_v2","table_update_row","table_update_rows_by_filter","table_upsert_row","tailscale_authorize_device","tailscale_create_auth_key","tailscale_delete_auth_key","tailscale_delete_device","tailscale_delete_user","tailscale_expire_device_key","tailscale_get_acl","tailscale_get_auth_key","tailscale_get_device","tailscale_get_device_routes","tailscale_get_dns_preferences","tailscale_get_dns_searchpaths","tailscale_list_auth_keys","tailscale_list_devices","tailscale_list_dns_nameservers","tailscale_list_users","tailscale_set_acl","tailscale_set_device_routes","tailscale_set_device_tags","tailscale_set_dns_nameservers","tailscale_set_dns_preferences","tailscale_set_dns_searchpaths","tailscale_suspend_user","tailscale_update_device_key","tavily_crawl","tavily_extract","tavily_map","tavily_search","telegram_copy_message","telegram_delete_message","telegram_edit_message_text","telegram_forward_message","telegram_get_chat","telegram_get_chat_member","telegram_message","telegram_pin_message","telegram_send_animation","telegram_send_audio","telegram_send_chat_action","telegram_send_contact","telegram_send_document","telegram_send_location","telegram_send_photo","telegram_send_poll","telegram_send_video","telegram_set_message_reaction","telegram_unpin_message","temporal_cancel_workflow","temporal_count_workflows","temporal_create_schedule","temporal_delete_schedule","temporal_describe_schedule","temporal_describe_task_queue","temporal_describe_workflow","temporal_get_workflow_history","temporal_list_schedules","temporal_list_workflows","temporal_pause_schedule","temporal_query_workflow","temporal_reset_workflow","temporal_signal_with_start","temporal_signal_workflow","temporal_start_workflow","temporal_terminate_workflow","temporal_trigger_schedule","temporal_unpause_schedule","temporal_update_workflow","textract_analyze_expense","textract_analyze_id","textract_parser","textract_parser_v2","thinking_tool","thrive_add_audience_managers","thrive_add_audience_members","thrive_add_user_tags","thrive_create_assignment","thrive_create_audience","thrive_create_completion","thrive_create_user","thrive_delete_assignment","thrive_delete_audience","thrive_delete_user","thrive_get_activity","thrive_get_assignment","thrive_get_audience","thrive_get_completion","thrive_get_content","thrive_get_cpd_category","thrive_get_cpd_entry","thrive_get_cpd_requirement","thrive_get_enrolment","thrive_get_skill_levels","thrive_get_tag","thrive_get_user_by_id","thrive_get_user_by_ref","thrive_list_assignments","thrive_list_audience_managers","thrive_list_audience_members","thrive_list_audiences","thrive_list_completions","thrive_list_enrolments","thrive_list_tags","thrive_query_activities","thrive_query_content","thrive_query_cpd_categories","thrive_query_cpd_entries","thrive_query_cpd_requirements","thrive_query_cpd_user_summaries","thrive_remove_audience_manager","thrive_remove_audience_member","thrive_remove_user_tags","thrive_replace_audience_managers","thrive_replace_audience_members","thrive_search_users","thrive_suspend_user","thrive_update_assignment","thrive_update_audience","thrive_update_user","thrive_update_user_skills","tiktok_get_post_status","tiktok_get_user","tiktok_list_videos","tiktok_query_videos","tiktok_upload_video_draft","tinybird_append_datasource","tinybird_delete_datasource_rows","tinybird_events","tinybird_get_job","tinybird_query","tinybird_query_pipe","tinybird_truncate_datasource","trello_add_checklist","trello_add_checklist_item","trello_add_comment","trello_add_label","trello_add_member","trello_create_board","trello_create_card","trello_create_list","trello_delete_card","trello_get_actions","trello_get_board","trello_get_card","trello_list_cards","trello_list_lists","trello_list_members","trello_remove_label","trello_remove_member","trello_search","trello_update_card","trello_update_checklist_item","trello_update_list","trigger_dev_activate_schedule","trigger_dev_add_run_tags","trigger_dev_batch_trigger_task","trigger_dev_cancel_run","trigger_dev_complete_waitpoint_token","trigger_dev_create_env_var","trigger_dev_create_schedule","trigger_dev_create_waitpoint_token","trigger_dev_deactivate_schedule","trigger_dev_delete_env_var","trigger_dev_delete_schedule","trigger_dev_execute_query","trigger_dev_get_batch","trigger_dev_get_batch_results","trigger_dev_get_deployment","trigger_dev_get_env_var","trigger_dev_get_latest_deployment","trigger_dev_get_query_schema","trigger_dev_get_queue","trigger_dev_get_run","trigger_dev_get_run_events","trigger_dev_get_run_result","trigger_dev_get_run_trace","trigger_dev_get_schedule","trigger_dev_get_waitpoint_token","trigger_dev_import_env_vars","trigger_dev_list_deployments","trigger_dev_list_env_vars","trigger_dev_list_queues","trigger_dev_list_runs","trigger_dev_list_schedules","trigger_dev_list_timezones","trigger_dev_list_waitpoint_tokens","trigger_dev_override_queue_concurrency","trigger_dev_pause_queue","trigger_dev_promote_deployment","trigger_dev_replay_run","trigger_dev_reschedule_run","trigger_dev_reset_queue_concurrency","trigger_dev_resume_queue","trigger_dev_trigger_task","trigger_dev_update_env_var","trigger_dev_update_run_metadata","trigger_dev_update_schedule","tts_azure","tts_cartesia","tts_deepgram","tts_elevenlabs","tts_google","tts_openai","tts_playht","twilio_send_sms","twilio_voice_get_recording","twilio_voice_list_calls","twilio_voice_make_call","typeform_create_form","typeform_delete_form","typeform_files","typeform_get_form","typeform_insights","typeform_list_forms","typeform_responses","typeform_update_form","upstash_redis_command","upstash_redis_delete","upstash_redis_exists","upstash_redis_expire","upstash_redis_get","upstash_redis_hget","upstash_redis_hgetall","upstash_redis_hset","upstash_redis_incr","upstash_redis_incrby","upstash_redis_keys","upstash_redis_lpush","upstash_redis_lrange","upstash_redis_set","upstash_redis_setnx","upstash_redis_ttl","uptimerobot_create_alert_contact","uptimerobot_create_maintenance_window","uptimerobot_create_monitor","uptimerobot_create_psp","uptimerobot_delete_alert_contact","uptimerobot_delete_maintenance_window","uptimerobot_delete_monitor","uptimerobot_delete_psp","uptimerobot_get_account","uptimerobot_get_alert_contact","uptimerobot_get_incident","uptimerobot_get_maintenance_window","uptimerobot_get_monitor","uptimerobot_get_psp","uptimerobot_list_alert_contacts","uptimerobot_list_incidents","uptimerobot_list_maintenance_windows","uptimerobot_list_monitors","uptimerobot_list_psps","uptimerobot_pause_monitor","uptimerobot_start_monitor","uptimerobot_update_maintenance_window","uptimerobot_update_monitor","uptimerobot_update_psp","vanta_download_document_file","vanta_get_control","vanta_get_document","vanta_get_framework","vanta_get_person","vanta_get_policy","vanta_get_risk_scenario","vanta_get_test","vanta_get_vendor","vanta_get_vulnerable_asset","vanta_list_control_documents","vanta_list_control_tests","vanta_list_controls","vanta_list_document_uploads","vanta_list_documents","vanta_list_framework_controls","vanta_list_frameworks","vanta_list_monitored_computers","vanta_list_people","vanta_list_policies","vanta_list_risk_scenarios","vanta_list_test_entities","vanta_list_tests","vanta_list_vendors","vanta_list_vulnerabilities","vanta_list_vulnerability_remediations","vanta_list_vulnerable_assets","vanta_submit_document","vanta_upload_document_file","vercel_add_domain","vercel_add_project_domain","vercel_cancel_deployment","vercel_create_alias","vercel_create_check","vercel_create_deployment","vercel_create_dns_record","vercel_create_edge_config","vercel_create_env_var","vercel_create_project","vercel_create_webhook","vercel_delete_alias","vercel_delete_deployment","vercel_delete_dns_record","vercel_delete_domain","vercel_delete_edge_config","vercel_delete_env_var","vercel_delete_project","vercel_delete_webhook","vercel_get_alias","vercel_get_check","vercel_get_deployment","vercel_get_deployment_events","vercel_get_domain","vercel_get_domain_config","vercel_get_edge_config","vercel_get_edge_config_items","vercel_get_env_vars","vercel_get_project","vercel_get_team","vercel_get_user","vercel_get_webhook","vercel_list_aliases","vercel_list_checks","vercel_list_deployment_files","vercel_list_deployments","vercel_list_dns_records","vercel_list_domains","vercel_list_edge_configs","vercel_list_project_domains","vercel_list_projects","vercel_list_team_members","vercel_list_teams","vercel_list_webhooks","vercel_pause_project","vercel_promote_deployment","vercel_remove_project_domain","vercel_rerequest_check","vercel_unpause_project","vercel_update_check","vercel_update_dns_record","vercel_update_edge_config_items","vercel_update_env_var","vercel_update_project","vercel_update_project_domain","vercel_verify_project_domain","video_falai","video_luma","video_minimax","video_runway","video_veo","vision_tool","vision_tool_v2","wealthbox_read_contact","wealthbox_read_note","wealthbox_read_task","wealthbox_write_contact","wealthbox_write_note","wealthbox_write_task","webflow_create_item","webflow_delete_item","webflow_get_item","webflow_list_items","webflow_update_item","webhook_request","whatsapp_get_media","whatsapp_mark_read","whatsapp_send_interactive","whatsapp_send_media","whatsapp_send_message","whatsapp_send_reaction","whatsapp_send_template","whatsapp_upload_media","wikipedia_content","wikipedia_random","wikipedia_search","wikipedia_summary","windchill_check_in_document","windchill_check_in_documents","windchill_check_out_document","windchill_check_out_documents","windchill_create_document","windchill_create_documents","windchill_delete_document","windchill_delete_documents","windchill_download_attachment","windchill_download_primary_content","windchill_get_document","windchill_get_document_structure","windchill_get_primary_content","windchill_get_valid_state_transitions","windchill_list_attachments","windchill_list_documents","windchill_revise_document","windchill_revise_documents","windchill_set_lifecycle_state","windchill_undo_check_out_document","windchill_undo_check_out_documents","windchill_update_common_properties","windchill_update_document","windchill_update_document_security_labels","windchill_update_documents","windchill_upload_attachments","windchill_upload_primary_content","wiza_company_enrichment","wiza_get_credits","wiza_individual_reveal","wiza_prospect_search","wordpress_create_category","wordpress_create_comment","wordpress_create_page","wordpress_create_post","wordpress_create_tag","wordpress_delete_category","wordpress_delete_comment","wordpress_delete_media","wordpress_delete_page","wordpress_delete_post","wordpress_delete_tag","wordpress_get_category","wordpress_get_current_user","wordpress_get_media","wordpress_get_page","wordpress_get_post","wordpress_get_tag","wordpress_get_user","wordpress_list_categories","wordpress_list_comments","wordpress_list_media","wordpress_list_pages","wordpress_list_posts","wordpress_list_tags","wordpress_list_users","wordpress_search_content","wordpress_update_category","wordpress_update_comment","wordpress_update_page","wordpress_update_post","wordpress_update_tag","wordpress_upload_media","workday_assign_onboarding","workday_change_job","workday_create_prehire","workday_get_compensation","workday_get_organizations","workday_get_worker","workday_hire_employee","workday_list_workers","workday_terminate_worker","workday_update_worker","workflow_executor","x_create_bookmark","x_create_tweet","x_delete_bookmark","x_delete_tweet","x_get_blocking","x_get_bookmarks","x_get_followers","x_get_following","x_get_liked_tweets","x_get_liking_users","x_get_me","x_get_personalized_trends","x_get_quote_tweets","x_get_retweeted_by","x_get_trends_by_woeid","x_get_tweets_by_ids","x_get_usage","x_get_user_mentions","x_get_user_timeline","x_get_user_tweets","x_hide_reply","x_manage_block","x_manage_follow","x_manage_like","x_manage_mute","x_manage_retweet","x_read","x_search","x_search_tweets","x_search_users","x_user","x_write","youtube_channel_info","youtube_channel_playlists","youtube_channel_videos","youtube_comments","youtube_playlist_items","youtube_search","youtube_trending","youtube_video_categories","youtube_video_details","zendesk_autocomplete_organizations","zendesk_create_organization","zendesk_create_organizations_bulk","zendesk_create_ticket","zendesk_create_tickets_bulk","zendesk_create_user","zendesk_create_users_bulk","zendesk_delete_organization","zendesk_delete_ticket","zendesk_delete_user","zendesk_get_current_user","zendesk_get_organization","zendesk_get_organizations","zendesk_get_ticket","zendesk_get_tickets","zendesk_get_user","zendesk_get_users","zendesk_merge_tickets","zendesk_search","zendesk_search_count","zendesk_search_users","zendesk_update_organization","zendesk_update_ticket","zendesk_update_tickets_bulk","zendesk_update_user","zendesk_update_users_bulk","zep_add_messages","zep_add_user","zep_create_thread","zep_delete_thread","zep_get_context","zep_get_messages","zep_get_threads","zep_get_user","zep_get_user_threads","zerobounce_get_credits","zerobounce_verify_email","zoho_desk_add_comment","zoho_desk_get_attachment","zoho_desk_get_contact","zoho_desk_get_thread","zoho_desk_get_ticket","zoho_desk_list_comments","zoho_desk_list_organizations","zoho_desk_list_threads","zoho_desk_list_tickets","zoho_desk_update_ticket","zoom_create_meeting","zoom_delete_meeting","zoom_delete_recording","zoom_get_meeting","zoom_get_meeting_invitation","zoom_get_meeting_recordings","zoom_list_meetings","zoom_list_past_participants","zoom_list_recordings","zoom_update_meeting","zoominfo_enrich_companies","zoominfo_enrich_contacts","zoominfo_search_companies","zoominfo_search_contacts","zoominfo_search_intent","zoominfo_search_news"]' ) export default toolIds diff --git a/apps/sim/tools/generated/tool-metadata.ts b/apps/sim/tools/generated/tool-metadata.ts index f76ac0d4b79..4c8c80724cc 100644 --- a/apps/sim/tools/generated/tool-metadata.ts +++ b/apps/sim/tools/generated/tool-metadata.ts @@ -3,7 +3,7 @@ /** Serializable metadata for every built-in tool, keyed by tool id. */ const toolMetadata: Record = JSON.parse( - '{"a2a_cancel_task":{"id":"a2a_cancel_task","name":"A2A Cancel Task","description":"Request cancellation of an in-progress A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to cancel"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}}},"a2a_get_agent_card":{"id":"a2a_get_agent_card","name":"A2A Get Agent Card","description":"Fetch the Agent Card (discovery document) for an external A2A agent.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}}},"a2a_get_task":{"id":"a2a_get_task","name":"A2A Get Task","description":"Retrieve the current state and result of an A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to retrieve"},"historyLength":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of history messages to include"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}}},"a2a_send_message":{"id":"a2a_send_message","name":"A2A Send Message","description":"Send a message to an external A2A agent and return its response.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"message":{"type":"string","required":true,"visibility":"user-or-llm","description":"The message text to send"},"data":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional structured JSON data to attach"},"files":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional files to attach"},"taskId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Existing task ID to continue"},"contextId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversation context ID to continue"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}}},"agentmail_create_draft":{"id":"agentmail_create_draft","name":"Create Draft","description":"Create a new email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to create the draft in"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"inReplyTo":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of message being replied to"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}}},"agentmail_create_inbox":{"id":"agentmail_create_inbox","name":"Create Inbox","description":"Create a new email inbox with AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"username":{"type":"string","required":false,"visibility":"user-or-llm","description":"Username for the inbox email address"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Domain for the inbox email address"},"displayName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Display name for the inbox"}}},"agentmail_delete_draft":{"id":"agentmail_delete_draft","name":"Delete Draft","description":"Delete an email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to delete"}}},"agentmail_delete_inbox":{"id":"agentmail_delete_inbox","name":"Delete Inbox","description":"Delete an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to delete"}}},"agentmail_delete_thread":{"id":"agentmail_delete_thread","name":"Delete Thread","description":"Delete an email thread in AgentMail (moves to trash, or permanently deletes if already in trash)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to delete"},"permanent":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Force permanent deletion instead of moving to trash"}}},"agentmail_forward_message":{"id":"agentmail_forward_message","name":"Forward Message","description":"Forward an email message to new recipients in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to forward"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional plain text to prepend"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional HTML to prepend"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}}},"agentmail_get_draft":{"id":"agentmail_get_draft","name":"Get Draft","description":"Get details of a specific email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox the draft belongs to"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to retrieve"}}},"agentmail_get_inbox":{"id":"agentmail_get_inbox","name":"Get Inbox","description":"Get details of a specific email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to retrieve"}}},"agentmail_get_message":{"id":"agentmail_get_message","name":"Get Message","description":"Get details of a specific email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to retrieve"}}},"agentmail_get_thread":{"id":"agentmail_get_thread","name":"Get Thread","description":"Get details of a specific email thread including messages in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to retrieve"}}},"agentmail_list_drafts":{"id":"agentmail_list_drafts","name":"List Drafts","description":"List email drafts in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list drafts from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of drafts to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}}},"agentmail_list_inboxes":{"id":"agentmail_list_inboxes","name":"List Inboxes","description":"List all email inboxes in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of inboxes to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}}},"agentmail_list_messages":{"id":"agentmail_list_messages","name":"List Messages","description":"List messages in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list messages from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of messages to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}}},"agentmail_list_threads":{"id":"agentmail_list_threads","name":"List Threads","description":"List email threads in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list threads from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of threads to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"},"labels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to filter threads by"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads after this ISO 8601 timestamp"}}},"agentmail_reply_message":{"id":"agentmail_reply_message","name":"Reply to Message","description":"Reply to an existing email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to reply from"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to reply to"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text reply body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML reply body"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override recipient email addresses (comma-separated)"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC email addresses (comma-separated)"},"replyAll":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reply to all recipients of the original message"}}},"agentmail_send_draft":{"id":"agentmail_send_draft","name":"Send Draft","description":"Send an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to send"}}},"agentmail_send_message":{"id":"agentmail_send_message","name":"Send Message","description":"Send an email message from an AgentMail inbox","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to send from"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email address (comma-separated for multiple)"},"subject":{"type":"string","required":true,"visibility":"user-or-llm","description":"Email subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text email body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML email body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}}},"agentmail_update_draft":{"id":"agentmail_update_draft","name":"Update Draft","description":"Update an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to update"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}}},"agentmail_update_inbox":{"id":"agentmail_update_inbox","name":"Update Inbox","description":"Update the display name of an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to update"},"displayName":{"type":"string","required":true,"visibility":"user-or-llm","description":"New display name for the inbox"}}},"agentmail_update_message":{"id":"agentmail_update_message","name":"Update Message","description":"Add or remove labels on an email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the message"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the message"}}},"agentmail_update_thread":{"id":"agentmail_update_thread","name":"Update Thread Labels","description":"Add or remove labels on an email thread in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the thread"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the thread"}}},"agentphone_create_call":{"id":"agentphone_create_call","name":"Create Outbound Call","description":"Initiate an outbound voice call from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent that will handle the call"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number to call in E.164 format (e.g. +14155551234)"},"fromNumberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to use as caller ID. Must belong to the agent. If omitted, the agent\'s first assigned number is used."},"initialGreeting":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional greeting spoken when the recipient answers"},"voice":{"type":"string","required":false,"visibility":"user-or-llm","description":"Voice ID override for this call (defaults to the agent\'s configured voice)"},"systemPrompt":{"type":"string","required":false,"visibility":"user-or-llm","description":"When provided, uses a built-in LLM for the conversation instead of forwarding to your webhook"}}},"agentphone_create_contact":{"id":"agentphone_create_contact","name":"Create Contact","description":"Create a new contact in AgentPhone","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"phoneNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number in E.164 format (e.g. +14155551234)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact\'s full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Contact\'s email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Freeform notes stored on the contact"}}},"agentphone_create_number":{"id":"agentphone_create_number","name":"Create Phone Number","description":"Provision a new SMS- and voice-enabled phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code (e.g. US, CA). Defaults to US."},"areaCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Preferred area code (US/CA only, e.g. \\"415\\"). Best-effort — may be ignored if unavailable."},"agentId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optionally attach the number to an agent immediately"}}},"agentphone_delete_contact":{"id":"agentphone_delete_contact","name":"Delete Contact","description":"Delete a contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}}},"agentphone_get_call":{"id":"agentphone_get_call","name":"Get Call","description":"Fetch a call and its full transcript","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve"}}},"agentphone_get_call_transcript":{"id":"agentphone_get_call_transcript","name":"Get Call Transcript","description":"Get the full ordered transcript for a call","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve the transcript for"}}},"agentphone_get_contact":{"id":"agentphone_get_contact","name":"Get Contact","description":"Fetch a single contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}}},"agentphone_get_conversation":{"id":"agentphone_get_conversation","name":"Get Conversation","description":"Get a conversation along with its recent messages","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"messageLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of recent messages to include (default 50, max 100)"}}},"agentphone_get_conversation_messages":{"id":"agentphone_get_conversation_messages","name":"Get Conversation Messages","description":"Get paginated messages for a conversation","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}}},"agentphone_get_number_messages":{"id":"agentphone_get_number_messages","name":"Get Phone Number Messages","description":"Fetch messages received on a specific phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}}},"agentphone_get_usage":{"id":"agentphone_get_usage","name":"Get Usage","description":"Retrieve current usage statistics for the AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"}}},"agentphone_get_usage_daily":{"id":"agentphone_get_usage_daily","name":"Get Daily Usage","description":"Get a daily breakdown of usage (messages, calls, webhooks) for the last N days","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"days":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of days to return (1-365, default 30)"}}},"agentphone_get_usage_monthly":{"id":"agentphone_get_usage_monthly","name":"Get Monthly Usage","description":"Get monthly usage aggregation (messages, calls, webhooks) for the last N months","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"months":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of months to return (1-24, default 6)"}}},"agentphone_list_calls":{"id":"agentphone_list_calls","name":"List Calls","description":"List voice calls for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"},"status":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by status (completed, in-progress, failed)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by direction (inbound, outbound)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by call type (pstn, web)"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search by phone number (matches fromNumber or toNumber)"}}},"agentphone_list_contacts":{"id":"agentphone_list_contacts","name":"List Contacts","description":"List contacts for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by name or phone number (case-insensitive contains)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 50, max 200)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}}},"agentphone_list_conversations":{"id":"agentphone_list_conversations","name":"List Conversations","description":"List conversations (message threads) for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}}},"agentphone_list_numbers":{"id":"agentphone_list_numbers","name":"List Phone Numbers","description":"List all phone numbers provisioned for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}}},"agentphone_react_to_message":{"id":"agentphone_react_to_message","name":"React to Message","description":"Send an iMessage tapback reaction to a message (iMessage only)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to react to"},"reaction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Reaction type: love, like, dislike, laugh, emphasize, or question"}}},"agentphone_release_number":{"id":"agentphone_release_number","name":"Release Phone Number","description":"Release (delete) a phone number. This action is irreversible.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number to release"}}},"agentphone_send_message":{"id":"agentphone_send_message","name":"Send Message","description":"Send an outbound SMS or iMessage from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent sending the message"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient phone number in E.164 format (e.g. +14155551234)"},"body":{"type":"string","required":true,"visibility":"user-or-llm","description":"Message text to send"},"mediaUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional URL of an image, video, or file to attach"},"numberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to send from. If omitted, the agent\'s first assigned number is used."}}},"agentphone_update_contact":{"id":"agentphone_update_contact","name":"Update Contact","description":"Update a contact\'s fields","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"New phone number in E.164 format"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New contact name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"New email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"New freeform notes"}}},"agentphone_update_conversation":{"id":"agentphone_update_conversation","name":"Update Conversation","description":"Update conversation metadata (stored state). Pass null to clear existing metadata.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"metadata":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom key-value metadata to store on the conversation. Pass null to clear existing metadata."}}},"agiloft_async_status":{"id":"agiloft_async_status","name":"Agiloft Async Status","description":"Check whether an asynchronous Agiloft call, such as a run action button, has completed.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table the asynchronous call was made against"},"callbackId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Callback ID returned by the asynchronous call, e.g. from Run Action Button"}}},"agiloft_attach_file":{"id":"agiloft_attach_file","name":"Agiloft Attach File","description":"Attach a file to a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to attach the file to"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"file":{"type":"file","required":true,"visibility":"user-or-llm","description":"File to attach"},"fileName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name to assign to the file (defaults to original file name)"},"overwrite":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Replace the contents of the field instead of adding another file to it"}}},"agiloft_attachment_info":{"id":"agiloft_attachment_info","name":"Agiloft Attachment Info","description":"Get information about file attachments on a record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to check attachments on"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field to inspect"}}},"agiloft_create_record":{"id":"agiloft_create_record","name":"Agiloft Create Record","description":"Create a new record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record field values as a JSON object (e.g., {\\"first_name\\": \\"John\\", \\"status\\": \\"Active\\"})"}}},"agiloft_delete_record":{"id":"agiloft_delete_record","name":"Agiloft Delete Record","description":"Delete a record from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to delete"},"substituteIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated IDs of records that adopt the dependants of the deleted record. Read only when the delete rule is REPLACE_WITH_ANOTHER."},"deleteRule":{"type":"string","required":false,"visibility":"user-or-llm","description":"How to treat records that depend on this one: ERROR_IF_DEPENDANTS (default — fails rather than cascading), APPLY_DELETE_WHERE_POSSIBLE, DELETE_WHERE_POSSIBLE_OTHERWISE_UNLINK, APPLY_UNLINK, UNLINK_WHERE_POSSIBLE_OTHERWISE_DELETE, or REPLACE_WITH_ANOTHER"}}},"agiloft_get_choice_line_id":{"id":"agiloft_get_choice_line_id","name":"Agiloft Get Choice Line ID","description":"Resolve the internal numeric ID of a choice-list value, for use in EWSelect WHERE clauses against choice fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"case\\", \\"contracts\\")"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice field name (e.g., \\"priority\\", \\"status\\")"},"value":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice display value to resolve (e.g., \\"High\\", \\"Active\\")"}}},"agiloft_list_tables":{"id":"agiloft_list_tables","name":"Agiloft List Tables","description":"List the tables and fields in an Agiloft knowledge base, to discover the logical names other operations need.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":false,"visibility":"user-or-llm","description":"Logical name of a single table to describe (e.g., \\"contacts\\"). Leave empty to list every table in the knowledge base."},"includeLinkedInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the source table and column behind each linked field"},"skipColumnsInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Return table names only, omitting field details, for a much smaller response"}}},"agiloft_lock_record":{"id":"agiloft_lock_record","name":"Agiloft Lock Record","description":"Lock, unlock, or check the lock status of an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to lock, unlock, or check"},"lockAction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Action to perform: \\"lock\\", \\"unlock\\", or \\"check\\""},"force":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Unlock only: release a lock held by another user."}}},"agiloft_nlp_search":{"id":"agiloft_nlp_search","name":"Agiloft Natural Language Search","description":"Search Agiloft records by describing what you want in plain language, such as \\"active NDAs submitted last month\\".","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"nlpQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The request in plain language, e.g. \\"Show me open, high-priority contracts\\". Structured field filters are not accepted — use Search Records for those."},"fields":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated field names to return, e.g. \\"id, contract_title1, company_name\\""},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number, starting from 0"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Records per page"}}},"agiloft_read_record":{"id":"agiloft_read_record","name":"Agiloft Read Record","description":"Read a record by ID from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to read"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the response"}}},"agiloft_remove_attachment":{"id":"agiloft_remove_attachment","name":"Agiloft Remove Attachment","description":"Remove an attached file from a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file to remove (starting from 0)"}}},"agiloft_retrieve_attachment":{"id":"agiloft_retrieve_attachment","name":"Agiloft Retrieve Attachment","description":"Download an attached file from an Agiloft record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file in the field (starting from 0)"}}},"agiloft_run_action_button":{"id":"agiloft_run_action_button","name":"Agiloft Run Action Button","description":"Run an action button on an Agiloft record, such as an approval or send-for-signature step.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"case\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to run the action button on"},"actionButtonField":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical name of the field holding the action button (e.g., \\"ab_field\\")"}}},"agiloft_saved_search":{"id":"agiloft_saved_search","name":"Agiloft Saved Search","description":"List the saved searches defined for an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical table name to list saved searches for (e.g., \\"contract\\")"}}},"agiloft_search_records":{"id":"agiloft_search_records","name":"Agiloft Search Records","description":"Search for records in an Agiloft table using a query.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name to search in (e.g., \\"contracts\\", \\"contacts.employees\\")"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Ad hoc EWSearch query. Combine conditions with && (and) or || (or) and quote every value — e.g. \\"summary~=\'test\'&&priority=\'High\'\\". Required unless a saved search is given."},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Label of a saved search defined on the table (e.g., \\"C: Status is Closed\\"). Can be combined with a query to narrow it further."},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the results"},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number for paginated results (starting from 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return per page. Agiloft treats 0 as \\"all records\\", so leave it unset or use a positive value to keep result sizes bounded."}}},"agiloft_select_records":{"id":"agiloft_select_records","name":"Agiloft Select Records","description":"Select record IDs matching a SQL WHERE clause from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"where":{"type":"string","required":true,"visibility":"user-or-llm","description":"SQL WHERE clause using database column names (e.g., \\"summary like \'%new%\'\\" or \\"assigned_person=\'John Doe\'\\"). EWSelect has no page size and returns every matching ID, so append a database limit such as \\"limit 0,200\\" to bound the result."}}},"agiloft_update_record":{"id":"agiloft_update_record","name":"Agiloft Update Record","description":"Update an existing record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to update"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Updated field values as a JSON object (e.g., {\\"status\\": \\"Active\\", \\"priority\\": \\"High\\"})"}}},"agiloft_upsert_record":{"id":"agiloft_upsert_record","name":"Agiloft Upsert Record","description":"Create an Agiloft record, or update it when a record already matches the given fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"match":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field used to find an existing record (e.g., \\"ext_id\\"). Pick something that identifies a record uniquely — if more than one record matches, Agiloft writes nothing and returns a conflict."},"async":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Queue the write instead of waiting for it. Returns a callback ID instead of a record ID; pass that to Async Status to poll the result."},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field values as a JSON object. On create these populate the new record; on update only the supplied fields change."}}},"ahrefs_anchors":{"id":"ahrefs_anchors","name":"Ahrefs Anchors","description":"Get the anchor text distribution for a target domain or URL\'s backlinks, showing how many links and referring domains use each anchor text.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_backlinks":{"id":"ahrefs_backlinks","name":"Ahrefs Backlinks","description":"Get a list of backlinks pointing to a target domain or URL. Returns details about each backlink including source URL, anchor text, and domain rating.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live backlinks), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_backlinks_stats":{"id":"ahrefs_backlinks_stats","name":"Ahrefs Backlinks Stats","description":"Get backlink and referring domain totals for a target domain or URL, both currently live and across all time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_batch_analysis":{"id":"ahrefs_batch_analysis","name":"Ahrefs Batch Analysis","description":"Get bulk SEO metrics (Domain Rating, backlinks, referring domains, organic traffic, and more) for multiple domains or URLs in a single request. Useful for comparing many competitors at once.","version":"1.0.0","params":{"targets":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated list of domains or URLs to analyze. Example: \\"example.com,competitor.com\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode applied to every target: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"protocol":{"type":"string","required":false,"visibility":"user-or-llm","description":"Protocol applied to every target: \\"both\\" (default), \\"http\\", or \\"https\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_broken_backlinks":{"id":"ahrefs_broken_backlinks","name":"Ahrefs Broken Backlinks","description":"Get a list of broken backlinks pointing to a target domain or URL. Useful for identifying link reclamation opportunities.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_domain_rating":{"id":"ahrefs_domain_rating","name":"Ahrefs Domain Rating","description":"Get the Domain Rating (DR) and Ahrefs Rank for a target domain. Domain Rating shows the strength of a website\'s backlink profile on a scale from 0 to 100.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze (e.g., example.com)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date for historical data in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_domain_rating_history":{"id":"ahrefs_domain_rating_history","name":"Ahrefs Domain Rating History","description":"Get the historical Domain Rating (DR) trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_keyword_overview":{"id":"ahrefs_keyword_overview","name":"Ahrefs Keyword Overview","description":"Get detailed metrics for a keyword including search volume, keyword difficulty, CPC, clicks, and traffic potential.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The keyword to analyze"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_keywords_history":{"id":"ahrefs_keywords_history","name":"Ahrefs Keywords History","description":"Get the historical organic keyword ranking distribution for a target domain or URL over a date range: how many keywords rank in each position bucket at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_metrics":{"id":"ahrefs_metrics","name":"Ahrefs Metrics","description":"Get a one-call organic and paid search overview for a target domain or URL: organic traffic, organic keywords, paid traffic, paid keywords, and estimated traffic cost.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_metrics_history":{"id":"ahrefs_metrics_history","name":"Ahrefs Metrics History","description":"Get the historical organic and paid traffic trend for a target domain or URL over a date range: organic traffic/cost and paid traffic/cost at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_organic_competitors":{"id":"ahrefs_organic_competitors","name":"Ahrefs Organic Competitors","description":"Get domains that compete with a target domain or URL for the same organic keywords, ranked by keyword overlap.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_organic_keywords":{"id":"ahrefs_organic_keywords","name":"Ahrefs Organic Keywords","description":"Get organic keywords that a target domain or URL ranks for in Google search results. Returns keyword details including search volume, ranking position, and estimated traffic.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_paid_pages":{"id":"ahrefs_paid_pages","name":"Ahrefs Paid Pages","description":"Get a target domain\'s pages that receive paid search traffic, sorted by estimated paid traffic. Returns page URLs with their paid traffic, keyword counts, and estimated spend.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_rank_tracker_competitors_overview":{"id":"ahrefs_rank_tracker_competitors_overview","name":"Ahrefs Rank Tracker Competitors Overview","description":"Get competitor rankings for the keywords tracked in an Ahrefs Rank Tracker project: each tracked keyword\'s volume and difficulty alongside every competitor\'s position, traffic, and traffic value. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_rank_tracker_competitors_stats":{"id":"ahrefs_rank_tracker_competitors_stats","name":"Ahrefs Rank Tracker Competitors Stats","description":"Get aggregate competitor stats for an Ahrefs Rank Tracker project: each competitor\'s traffic, traffic value, average position, and share of voice across all tracked keywords. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report metrics for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_rank_tracker_overview":{"id":"ahrefs_rank_tracker_overview","name":"Ahrefs Rank Tracker Overview","description":"Get ranking overview metrics for the keywords tracked in an Ahrefs Rank Tracker project: position, search volume, keyword difficulty, and estimated traffic. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_rank_tracker_serp_overview":{"id":"ahrefs_rank_tracker_serp_overview","name":"Ahrefs Rank Tracker SERP Overview","description":"Get the full SERP (search engine results page) for a keyword tracked in an Ahrefs Rank Tracker project, including every ranking URL with its position, title, and authority metrics. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The tracked keyword to retrieve SERP data for"},"country":{"type":"string","required":true,"visibility":"user-or-llm","description":"Country code for the tracked keyword. Example: \\"us\\", \\"gb\\", \\"de\\""},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"topPositions":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of top organic positions to return (defaults to all available)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Timestamp to return the last available SERP Overview at, in YYYY-MM-DDThh:mm:ss format"},"locationId":{"type":"number","required":false,"visibility":"user-or-llm","description":"Location ID of the tracked keyword, if tracked at a specific location"},"languageCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code of the tracked keyword"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_refdomains_history":{"id":"ahrefs_refdomains_history","name":"Ahrefs Referring Domains History","description":"Get the historical referring domains trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_referring_domains":{"id":"ahrefs_referring_domains","name":"Ahrefs Referring Domains","description":"Get a list of domains that link to a target domain or URL. Returns unique referring domains with their domain rating, backlink counts, and discovery dates.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost domains), or \\"since:YYYY-MM-DD\\" (domains found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_related_terms":{"id":"ahrefs_related_terms","name":"Ahrefs Related Terms","description":"Get keyword ideas related to a seed keyword: terms the same top-ranking pages also rank for (\\"also rank for\\") or also discuss (\\"also talk about\\"), with volume, difficulty, and CPC.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The seed keyword to find related terms for"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"terms":{"type":"string","required":false,"visibility":"user-or-llm","description":"Type of related keywords to return: \\"also_rank_for\\", \\"also_talk_about\\", or \\"all\\" (default: \\"all\\")"},"viewFor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Whether to derive related terms from the top 10 or top 100 ranking pages (default: \\"top_10\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_site_audit_page_explorer":{"id":"ahrefs_site_audit_page_explorer","name":"Ahrefs Site Audit Page Explorer","description":"Get crawled pages from an Ahrefs Site Audit project with health and SEO metrics: HTTP status, title, link counts, backlinks, indexability, and traffic. Optionally filter to pages affected by a specific issue.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Site Audit project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Crawl date in YYYY-MM-DDThh:mm:ss format (defaults to the most recent crawl)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip, for pagination"},"issueId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Only return pages affected by this issue ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"ahrefs_top_pages":{"id":"ahrefs_top_pages","name":"Ahrefs Top Pages","description":"Get the top pages of a target domain sorted by organic traffic. Returns page URLs with their traffic, keyword counts, and estimated traffic value.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}}},"airtable_create_records":{"id":"airtable_create_records","name":"Airtable Create Records","description":"Write new records to an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to create, each with a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_delete_records":{"id":"airtable_delete_records","name":"Airtable Delete Records","description":"Delete one or more records from an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordIds":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of record IDs to delete (each starts with \\"rec\\", e.g., [\\"recXXXXXXXXXXXXXX\\"]). Pass a single-element array to delete one record."}},"oauth":{"required":true,"provider":"airtable"}},"airtable_get_base_schema":{"id":"airtable_get_base_schema","name":"Airtable Get Base Schema","description":"Get the schema of all tables, fields, and views in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_get_record":{"id":"airtable_get_record","name":"Airtable Get Record","description":"Retrieve a single record from an Airtable table by its ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to retrieve (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_list_bases":{"id":"airtable_list_bases","name":"Airtable List Bases","description":"List all bases the authenticated user has access to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination offset for retrieving additional bases"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_list_records":{"id":"airtable_list_records","name":"Airtable List Records","description":"Read records from an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"maxRecords":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return (default: all records)"},"filterFormula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Formula to filter records (e.g., \\"({Field Name} = \'Value\')\\")"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_list_tables":{"id":"airtable_list_tables","name":"Airtable List Tables","description":"List all tables and their schema in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_update_multiple_records":{"id":"airtable_update_multiple_records","name":"Airtable Update Multiple Records","description":"Update multiple existing records in an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to update, each with an `id` and a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_update_record":{"id":"airtable_update_record","name":"Airtable Update Record","description":"Update an existing record in an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to update (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"},"fields":{"type":"json","required":true,"visibility":"user-or-llm","description":"An object containing the field names and their new values"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"}},"airtable_upsert_records":{"id":"airtable_upsert_records","name":"Airtable Upsert Records","description":"Update existing records or create new ones in an Airtable table, matching on the specified merge fields","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to upsert, each with a `fields` object"},"fieldsToMergeOn":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of field names used to match existing records (max 3). A record is updated when all merge fields match, otherwise it is created. Example: [\\"Name\\"]"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"}},"airweave_search":{"id":"airweave_search","name":"Airweave Search","description":"Search your synced data collections using Airweave. Supports semantic search with hybrid, neural, or keyword retrieval strategies. Optionally generate AI-powered answers from search results.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Airweave API Key for authentication"},"collectionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The readable ID of the collection to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query text"},"limit":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 100)"},"retrievalStrategy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retrieval strategy: hybrid (default), neural, or keyword"},"expandQuery":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate query variations to improve recall"},"rerank":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reorder results for improved relevance using LLM"},"generateAnswer":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate a natural-language answer to the query"}}},"algolia_add_record":{"id":"algolia_add_record","name":"Algolia Add Record","description":"Add or replace a record in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":false,"visibility":"user-or-llm","description":"Object ID for the record (auto-generated if not provided)"},"record":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object representing the record to add"}}},"algolia_batch_operations":{"id":"algolia_batch_operations","name":"Algolia Batch Operations","description":"Perform batch add, update, partial update, or delete operations on records in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of batch operations. Each item has \\"action\\" (addObject, updateObject, partialUpdateObject, partialUpdateObjectNoCreate, deleteObject, delete, clear) and \\"body\\" (the record data; must include objectID for update/delete; use an empty object {} for the index-level delete/clear actions)"}}},"algolia_browse_records":{"id":"algolia_browse_records","name":"Algolia Browse Records","description":"Browse and iterate over all records in an Algolia index using cursor pagination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key (must have browse ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to browse"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search query to filter browsed records"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string to narrow down results"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 1000, max: 1000)"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous browse response for pagination"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}}},"algolia_clear_records":{"id":"algolia_clear_records","name":"Algolia Clear Records","description":"Clear all records from an Algolia index while keeping settings, synonyms, and rules","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to clear"}}},"algolia_copy_move_index":{"id":"algolia_copy_move_index","name":"Algolia Copy/Move Index","description":"Copy or move an Algolia index to a new destination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the source index"},"operation":{"type":"string","required":true,"visibility":"user-or-llm","description":"Operation to perform: \\"copy\\" or \\"move\\""},"destination":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the destination index"},"scope":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of scopes to copy (only for \\"copy\\" operation): [\\"settings\\", \\"synonyms\\", \\"rules\\"]. Omit to copy everything including records."}}},"algolia_delete_by_filter":{"id":"algolia_delete_by_filter","name":"Algolia Delete By Filter","description":"Delete all records matching a filter from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter expression to match records for deletion (e.g., \\"category:outdated\\")"},"facetFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of facet filters (e.g., [\\"brand:Acme\\"])"},"numericFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of numeric filters (e.g., [\\"price > 100\\"])"},"tagFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of tag filters using the _tags attribute (e.g., [\\"published\\"])"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search filter (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search filter"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search filter"}}},"algolia_delete_index":{"id":"algolia_delete_index","name":"Algolia Delete Index","description":"Delete an entire Algolia index and all its records","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to delete"}}},"algolia_delete_record":{"id":"algolia_delete_record","name":"Algolia Delete Record","description":"Delete a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to delete"}}},"algolia_get_record":{"id":"algolia_get_record","name":"Algolia Get Record","description":"Get a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to retrieve"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"}}},"algolia_get_records":{"id":"algolia_get_records","name":"Algolia Get Records","description":"Retrieve multiple records by objectID from one or more Algolia indices","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Default index name for all requests"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of objects specifying records to retrieve. Each must have \\"objectID\\" and optionally \\"indexName\\" and \\"attributesToRetrieve\\"."}}},"algolia_get_settings":{"id":"algolia_get_settings","name":"Algolia Get Settings","description":"Retrieve the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"}}},"algolia_get_task_status":{"id":"algolia_get_task_status","name":"Algolia Get Task Status","description":"Check whether an Algolia indexing task has finished publishing","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index the task ran against"},"taskID":{"type":"number","required":true,"visibility":"user-or-llm","description":"The taskID returned by a previous write operation"}}},"algolia_list_indices":{"id":"algolia_list_indices","name":"Algolia List Indices","description":"List all indices in an Algolia application","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for paginating indices (default: not paginated)"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of indices per page (default: 100)"}}},"algolia_partial_update_record":{"id":"algolia_partial_update_record","name":"Algolia Partial Update Record","description":"Partially update a record in an Algolia index without replacing it entirely","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to update"},"attributes":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with attributes to update. Supports built-in operations like {\\"stock\\": {\\"_operation\\": \\"Decrement\\", \\"value\\": 1}}"},"createIfNotExists":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to create the record if it does not exist (default: true)"}}},"algolia_search":{"id":"algolia_search","name":"Algolia Search","description":"Search an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"Search query text"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 20)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number to retrieve (default: 0)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string (e.g., \\"category:electronics AND price < 100\\")"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"facets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of facet attribute names to retrieve counts for (use \\"*\\" for all)"},"getRankingInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to include detailed ranking information in each hit"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}}},"algolia_update_settings":{"id":"algolia_update_settings","name":"Algolia Update Settings","description":"Update the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have editSettings ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"settings":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with settings to update (e.g., {\\"searchableAttributes\\": [\\"name\\", \\"description\\"], \\"customRanking\\": [\\"desc(popularity)\\"]})"},"forwardToReplicas":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to apply changes to replica indices (default: false)"}}},"amplitude_event_segmentation":{"id":"amplitude_event_segmentation","name":"Amplitude Event Segmentation","description":"Query event analytics data with segmentation. Get event counts, uniques, averages, and more.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Event type name to analyze"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: uniques, totals, pct_dau, average, histogram, sums, value_avg, or formula (default: uniques)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (prefix custom user properties with \\"gp:\\")"},"groupBy2":{"type":"string","required":false,"visibility":"user-or-llm","description":"Second property name to group by (prefix custom user properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (max 1000)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON array of filter objects applied to the event, e.g. [{\\"subprop_type\\":\\"event\\",\\"subprop_key\\":\\"city\\",\\"subprop_op\\":\\"is\\",\\"subprop_value\\":[\\"San Francisco\\"]}]"},"formula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when metric is \\"formula\\", e.g. \\"UNIQUES(A)/UNIQUES(B)\\""},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_funnels":{"id":"amplitude_funnels","name":"Amplitude Funnels","description":"Analyze conversion rates and drop-off between a sequence of events.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"events":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON array of event objects, one per funnel step in order, e.g. [{\\"event_type\\":\\"signup\\"},{\\"event_type\\":\\"purchase\\"}]"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Funnel ordering: \\"ordered\\", \\"unordered\\", or \\"sequential\\" (default: ordered)"},"userType":{"type":"string","required":false,"visibility":"user-or-llm","description":"User type: \\"new\\" or \\"active\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: -300000 (real-time), -3600000 (hourly), 1 (daily), 7 (weekly), or 30 (monthly)"},"conversionWindowSeconds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversion window in seconds (default: 2592000, i.e. 30 days)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (default: 100, max: 1000)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_get_active_users":{"id":"amplitude_get_active_users","name":"Amplitude Get Active Users","description":"Get active or new user counts over a date range from the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: \\"active\\" or \\"new\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_get_revenue":{"id":"amplitude_get_revenue","name":"Amplitude Get Revenue","description":"Get revenue LTV data including ARPU, ARPPU, total revenue, and paying user counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric: 0 (ARPU), 1 (ARPPU), 2 (Total Revenue), 3 (Paying Users)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (limit: one)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_group_identify":{"id":"amplitude_group_identify","name":"Amplitude Group Identify","description":"Set group-level properties in Amplitude. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"groupType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Group classification (e.g., \\"company\\", \\"org_id\\")"},"groupValue":{"type":"string","required":true,"visibility":"user-or-llm","description":"Specific group identifier (e.g., \\"Acme Corp\\")"},"groupProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of group properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_identify_user":{"id":"amplitude_identify_user","name":"Amplitude Identify User","description":"Set user properties in Amplitude using the Identify API. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"userProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of user properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_list_events":{"id":"amplitude_list_events","name":"Amplitude List Events","description":"List all event types in the Amplitude project with their weekly totals and unique counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_realtime_active_users":{"id":"amplitude_realtime_active_users","name":"Amplitude Real-time Active Users","description":"Get real-time active user counts at 5-minute granularity for the last 2 days.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_retention":{"id":"amplitude_retention","name":"Amplitude Retention","description":"Measure how many users return to perform an action after a starting action.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"startEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON starting event object, e.g. {\\"event_type\\":\\"_new\\"} or {\\"event_type\\":\\"_active\\"}"},"returnEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON returning event object, e.g. {\\"event_type\\":\\"_all\\"} or {\\"event_type\\":\\"_active\\"}"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"retentionMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retention type: \\"bracket\\", \\"rolling\\", or \\"n-day\\" (default: n-day)"},"retentionBrackets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when Retention Mode is \\"bracket\\". Day ranges, e.g. [[0,4]]"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_send_event":{"id":"amplitude_send_event","name":"Amplitude Send Event","description":"Track an event in Amplitude using the HTTP V2 API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the event (e.g., \\"page_view\\", \\"purchase\\")"},"eventProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of custom event properties"},"userProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of user properties to set (supports $set, $setOnce, $add, $append, $unset)"},"time":{"type":"string","required":false,"visibility":"user-or-llm","description":"Event timestamp in milliseconds since epoch"},"sessionId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Session start time in milliseconds since epoch"},"insertId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Unique ID for deduplication (within 7-day window)"},"appVersion":{"type":"string","required":false,"visibility":"user-or-llm","description":"Application version string"},"platform":{"type":"string","required":false,"visibility":"user-or-llm","description":"Platform (e.g., \\"Web\\", \\"iOS\\", \\"Android\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code"},"language":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code (e.g., \\"en\\")"},"ip":{"type":"string","required":false,"visibility":"user-or-llm","description":"IP address for geo-location"},"price":{"type":"string","required":false,"visibility":"user-or-llm","description":"Price of the item purchased"},"quantity":{"type":"string","required":false,"visibility":"user-or-llm","description":"Quantity of items purchased"},"revenue":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue amount"},"productId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Product identifier"},"revenueType":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue type (e.g., \\"purchase\\", \\"refund\\")"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_user_activity":{"id":"amplitude_user_activity","name":"Amplitude User Activity","description":"Get the event stream for a specific user by their Amplitude ID.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"amplitudeId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Amplitude internal user ID"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Offset for pagination (default 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of events to return (default 1000, max 1000)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort direction: \\"latest\\" or \\"earliest\\" (default: latest)"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"amplitude_user_profile":{"id":"amplitude_user_profile","name":"Amplitude User Profile","description":"Get a user profile including properties, cohort memberships, and computed properties. Not available for EU data-residency projects.","version":"1.0.0","params":{"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"External user ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"getAmpProps":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include Amplitude user properties (true/false, default: false)"},"getCohortIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include cohort IDs the user belongs to (true/false, default: false)"},"getComputations":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include computed user properties (true/false, default: false)"}}},"amplitude_user_search":{"id":"amplitude_user_search","name":"Amplitude User Search","description":"Search for a user by User ID, Device ID, or Amplitude ID using the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"user":{"type":"string","required":true,"visibility":"user-or-llm","description":"User ID, Device ID, or Amplitude ID to search for"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}}},"apify_get_dataset_items":{"id":"apify_get_dataset_items","name":"APIFY Get Dataset Items","description":"Retrieve items stored in an APIFY dataset","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"datasetId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Dataset ID to read items from. Example: \\"9RnD3Pql2vGZkc5H5\\""},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max items to return (1-250000). Default: all items. Example: 500"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to skip at the start. Default: 0"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of fields to include. Example: \\"title,url,price\\""}}},"apify_get_run":{"id":"apify_get_run","name":"APIFY Get Run","description":"Get the status and details of an APIFY actor run","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"runId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor run ID to fetch. Example: \\"HG7ML7M8z78YcAPEB\\""}}},"apify_run_actor_async":{"id":"apify_run_actor_async","name":"APIFY Run Actor (Async)","description":"Run an APIFY actor asynchronously with polling for long-running tasks","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"waitForFinish":{"type":"number","required":false,"visibility":"user-or-llm","description":"Initial wait time in seconds (0-60) before polling starts. Example: 30"},"itemLimit":{"type":"number","required":false,"default":100,"visibility":"user-or-llm","description":"Max dataset items to fetch (1-250000). Default: 100. Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}}},"apify_run_actor_sync":{"id":"apify_run_actor_sync","name":"APIFY Run Actor (Sync)","description":"Run an APIFY actor synchronously and get results (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}}},"apify_run_task":{"id":"apify_run_task","name":"APIFY Run Task","description":"Run a saved APIFY actor task synchronously and get dataset items (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task ID or username/task-name. Examples: \\"janedoe/my-task\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON string that overrides the task\'s saved input. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}]}"},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max dataset items to return (1-250000). Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the run (128-32768). Example: 1024 for 1GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the run. Example: 300 for 5 minutes"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\""}}},"apollo_account_bulk_create":{"id":"apollo_account_bulk_create","name":"Apollo Bulk Create Accounts","description":"Create up to 100 accounts at once in your Apollo database. Set run_dedupe=true to deduplicate by domain, organization_id, and name. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"accounts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of accounts to create (max 100). Each account should include a name, and may optionally include domain, phone, phone_status_cd, raw_address, owner_id, linkedin_url, facebook_url, twitter_url, salesforce_id, and hubspot_id."},"append_label_names":{"type":"array","required":false,"visibility":"user-only","description":"Array of label names to add to ALL accounts in this request"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, performs aggressive deduplication by domain, organization_id, and name (defaults to false)"}}},"apollo_account_bulk_update":{"id":"apollo_account_bulk_update","name":"Apollo Bulk Update Accounts","description":"Update up to 1000 existing accounts at once in your Apollo database (higher limit than contacts!). Each account must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"account_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of account IDs to update with the same values (max 1000). Use with name/owner_id for uniform updates. Use either this OR account_attributes."},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this name to all accounts"},"owner_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this owner to all accounts"},"account_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this account stage to all accounts"},"account_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of account objects with individual updates (each must include id). Example: [{\\"id\\": \\"acc1\\", \\"name\\": \\"Acme\\", \\"owner_id\\": \\"u1\\", \\"account_stage_id\\": \\"s1\\", \\"typed_custom_fields\\": {\\"field_id\\": \\"value\\"}}]"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, processes the update asynchronously. Only supported when using account_ids; returns 422 if used with account_attributes."}}},"apollo_account_create":{"id":"apollo_account_create","name":"Apollo Create Account","description":"Create a new account (company) in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain without www. prefix (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the account"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}}},"apollo_account_search":{"id":"apollo_account_search","name":"Apollo Search Accounts","description":"Search your team\'s accounts in Apollo. Display limit: 50,000 records (100 records per page, 500 pages max). Use filters to narrow results. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter accounts by organization name (partial-match search)"},"account_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account stage IDs"},"account_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account label IDs"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"account_last_activity_date\\", \\"account_created_at\\", or \\"account_updated_at\\""},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Sort ascending when true. Defaults to descending."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"apollo_account_update":{"id":"apollo_account_update","name":"Apollo Update Account","description":"Update an existing account in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the account to update (e.g., \\"acc_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company phone number"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}}},"apollo_contact_bulk_create":{"id":"apollo_contact_bulk_create","name":"Apollo Bulk Create Contacts","description":"Create up to 100 contacts at once in your Apollo database. Supports deduplication to prevent creating duplicate contacts. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contacts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contacts to create (max 100). Each contact may include first_name, last_name, email, title, organization_name, account_id, owner_id, contact_stage_id, linkedin_url, phone (single string) or phone_numbers (array of {raw_number, position}), contact_emails, typed_custom_fields, and CRM IDs (salesforce_contact_id, hubspot_id, team_id) for cross-system matching"},"append_label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Label names to add to all contacts in this request (e.g., [\\"Hot Lead\\"])"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"Enable deduplication to prevent creating duplicate contacts. When true, existing contacts are returned without modification"}}},"apollo_contact_bulk_update":{"id":"apollo_contact_bulk_update","name":"Apollo Bulk Update Contacts","description":"Update up to 100 existing contacts at once in your Apollo database. Each contact must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to update. Must be paired with an object-form contact_attributes specifying the fields to apply uniformly to all listed contacts."},"contact_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Required. Either an array of per-contact updates (each with id) — used standalone — or a single object of attributes to apply to all contact_ids. Supported fields: owner_id, email, organization_name, title, first_name, last_name, account_id, present_raw_address, linkedin_url, typed_custom_fields"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"Force asynchronous processing. Automatically enabled for >100 contacts"}}},"apollo_contact_create":{"id":"apollo_contact_create","name":"Apollo Create Contact","description":"Create a new contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the contact"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID to associate with (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for POST /contacts)"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, Apollo deduplicates against existing contacts"}}},"apollo_contact_search":{"id":"apollo_contact_search","name":"Apollo Search Contacts","description":"Search your team\'s contacts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"contact_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by contact stage IDs"},"contact_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by Apollo label IDs (lists)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-only","description":"Sort field: contact_last_activity_date, contact_email_last_opened_at, contact_email_last_clicked_at, contact_created_at, or contact_updated_at"},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, sort ascending. Must be used together with sort_by_field"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"apollo_contact_update":{"id":"apollo_contact_update","name":"Apollo Update Contact","description":"Update an existing contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"contact_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the contact to update (e.g., \\"con_abc123\\")"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for PATCH /contacts/{id})"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"}}},"apollo_email_accounts":{"id":"apollo_email_accounts","name":"Apollo Get Email Accounts","description":"Get list of team\'s linked email accounts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"}}},"apollo_opportunity_create":{"id":"apollo_opportunity_create","name":"Apollo Create Opportunity","description":"Create a new deal for an account in your Apollo database (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of the account this opportunity belongs to (e.g., \\"acc_abc123\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}}},"apollo_opportunity_get":{"id":"apollo_opportunity_get","name":"Apollo Get Opportunity","description":"Retrieve complete details of a specific deal/opportunity by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to retrieve (e.g., \\"opp_abc123\\")"}}},"apollo_opportunity_search":{"id":"apollo_opportunity_search","name":"Apollo Search Opportunities","description":"Search and list all deals/opportunities in your team\'s Apollo account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"amount\\", \\"is_closed\\", or \\"is_won\\""},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"apollo_opportunity_update":{"id":"apollo_opportunity_update","name":"Apollo Update Opportunity","description":"Update an existing deal/opportunity in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to update (e.g., \\"opp_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}}},"apollo_organization_bulk_enrich":{"id":"apollo_organization_bulk_enrich","name":"Apollo Bulk Organization Enrichment","description":"Enrich data for up to 10 organizations at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domains":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of company domains to enrich (max 10, no www. or @, e.g., [\\"apollo.io\\", \\"stripe.com\\"])"}}},"apollo_organization_enrich":{"id":"apollo_organization_enrich","name":"Apollo Organization Enrichment","description":"Enrich data for a single organization using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domain":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"}}},"apollo_organization_search":{"id":"apollo_organization_search","name":"Apollo Organization Search","description":"Search Apollo\'s database for companies using filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company HQ locations (cities, US states, or countries)"},"organization_not_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Exclude companies whose HQ is in these locations"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges as \\"min,max\\" strings (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"q_organization_keyword_tags":{"type":"array","required":false,"visibility":"user-or-llm","description":"Industry or keyword tags"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Organization name to search for (e.g., \\"Acme\\", \\"TechCorp\\")"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to include (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Domain names to filter by (no www. or @, up to 1,000)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"apollo_people_bulk_enrich":{"id":"apollo_people_bulk_enrich","name":"Apollo Bulk People Enrichment","description":"Enrich data for up to 10 people at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"people":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of people to enrich (max 10)"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}}},"apollo_people_enrich":{"id":"apollo_people_enrich","name":"Apollo People Enrichment","description":"Enrich data for a single person using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the person"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the person"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Full name of the person (alternative to first_name/last_name)"},"id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the person"},"hashed_email":{"type":"string","required":false,"visibility":"user-or-llm","description":"MD5 or SHA-256 hashed email"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the person"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name where the person works"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"},"linkedin_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}}},"apollo_people_search":{"id":"apollo_people_search","name":"Apollo People Search","description":"Search Apollo\'s database for people using demographic filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"person_titles":{"type":"array","required":false,"visibility":"user-or-llm","description":"Job titles to search for (e.g., [\\"CEO\\", \\"VP of Sales\\"])"},"include_similar_titles":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to return people with job titles similar to person_titles"},"person_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Locations to search in (e.g., [\\"San Francisco, CA\\", \\"New York, NY\\"])"},"person_seniorities":{"type":"array","required":false,"visibility":"user-or-llm","description":"Seniority levels (one of: owner, founder, c_suite, partner, vp, head, director, manager, senior, entry, intern)"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to filter by (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"organization_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company names to search within (legacy filter)"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Headquarters locations of the people\'s current employer (e.g., [\'texas\', \'tokyo\', \'spain\'])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employer domain names (e.g., [\\"apollo.io\\", \\"microsoft.com\\"]) — up to 1,000, no www. or @"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges for the person\'s current employer. Each entry is \\"min,max\\" (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"contact_email_status":{"type":"array","required":false,"visibility":"user-or-llm","description":"Email statuses to filter by: \\"verified\\", \\"unverified\\", \\"likely to engage\\", \\"unavailable\\""},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination, default 1 (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, default 25, max 100 (e.g., 25, 50, 100)"}}},"apollo_sequence_add_contacts":{"id":"apollo_sequence_add_contacts","name":"Apollo Add Contacts to Sequence","description":"Add contacts to an Apollo sequence","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sequence_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the sequence to add contacts to (e.g., \\"seq_abc123\\")"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to add to the sequence (e.g., [\\"con_abc123\\", \\"con_def456\\"]). Either contact_ids or label_names must be provided."},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of label names to identify contacts to add to the sequence. Either contact_ids or label_names must be provided."},"send_email_from_email_account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the email account to send from. Use the Get Email Accounts operation to look this up."},"send_email_from_email_address":{"type":"string","required":false,"visibility":"user-only","description":"Specific email address to send from within the email account."},"sequence_no_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they have no email address"},"sequence_unverified_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts with unverified email addresses"},"sequence_job_change":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who recently changed jobs"},"sequence_active_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts active in other campaigns"},"sequence_finished_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who finished other campaigns"},"sequence_same_company_in_same_campaign":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if others from the same company are in the sequence"},"contacts_without_ownership_permission":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts without ownership permission"},"add_if_in_queue":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they are in the queue"},"contact_verification_skipped":{"type":"boolean","required":false,"visibility":"user-only","description":"Skip contact verification when adding"},"user_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the user performing the action"},"status":{"type":"string","required":false,"visibility":"user-only","description":"Initial status for added contacts: \\"active\\" or \\"paused\\""},"auto_unpause_at":{"type":"string","required":false,"visibility":"user-only","description":"ISO 8601 datetime to automatically unpause contacts"}}},"apollo_sequence_search":{"id":"apollo_sequence_search","name":"Apollo Search Sequences","description":"Search for sequences/campaigns in your team\'s Apollo account (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search sequences by name (e.g., \\"Outbound Q1\\", \\"Follow-up\\")"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"apollo_task_create":{"id":"apollo_task_create","name":"Apollo Create Task","description":"Create one or more tasks in Apollo (one task per contact_id, master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"user_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the Apollo user the task is assigned to"},"contact_ids":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contact IDs. One task is created per contact."},"priority":{"type":"string","required":false,"visibility":"user-or-llm","description":"Task priority: \\"high\\", \\"medium\\", or \\"low\\" (defaults to \\"medium\\")"},"due_at":{"type":"string","required":true,"visibility":"user-or-llm","description":"Due date/time in ISO 8601 format (e.g., \\"2024-12-31T23:59:59Z\\")"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task type: \\"call\\", \\"outreach_manual_email\\", \\"linkedin_step_connect\\", \\"linkedin_step_message\\", \\"linkedin_step_view_profile\\", \\"linkedin_step_interact_post\\", or \\"action_item\\""},"status":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task status: \\"scheduled\\", \\"completed\\", or \\"skipped\\""},"note":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-form note providing context for the task"}}},"apollo_task_search":{"id":"apollo_task_search","name":"Apollo Search Tasks","description":"Search for tasks in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"task_due_at\\" or \\"task_priority\\""},"open_factor_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Filter by status. Common values: [\\"task_types\\"] for open tasks, [\\"task_completed_at\\"] for completed tasks."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}}},"appconfig_create_application":{"id":"appconfig_create_application","name":"AppConfig Create Application","description":"Create an application in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the application to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the application"}}},"appconfig_create_configuration_profile":{"id":"appconfig_create_configuration_profile","name":"AppConfig Create Configuration Profile","description":"Create a configuration profile in an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the configuration profile in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the configuration profile"},"locationUri":{"type":"string","required":true,"visibility":"user-or-llm","description":"Where the configuration is stored. Use \\"hosted\\" for AppConfig-hosted configurations, or an SSM/S3 URI"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"ARN of an IAM role to retrieve the configuration (required for non-hosted URIs)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Profile type: AWS.Freeform (default) or AWS.AppConfig.FeatureFlags"}}},"appconfig_create_environment":{"id":"appconfig_create_environment","name":"AppConfig Create Environment","description":"Create an environment for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the environment in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the environment to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the environment"}}},"appconfig_create_hosted_configuration_version":{"id":"appconfig_create_hosted_configuration_version","name":"AppConfig Create Hosted Configuration Version","description":"Create a new hosted configuration version for an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to add the version to"},"content":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration content (e.g., a JSON or YAML document)"},"contentType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Content type of the configuration (e.g., application/json, text/plain)"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration version"},"latestVersionNumber":{"type":"number","required":false,"visibility":"user-or-llm","description":"The version number of the latest version, used for optimistic concurrency"},"versionLabel":{"type":"string","required":false,"visibility":"user-or-llm","description":"A user-defined label for the configuration version"}}},"appconfig_delete_application":{"id":"appconfig_delete_application","name":"AppConfig Delete Application","description":"Delete an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to delete"}}},"appconfig_delete_configuration_profile":{"id":"appconfig_delete_configuration_profile","name":"AppConfig Delete Configuration Profile","description":"Delete an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to delete"}}},"appconfig_delete_environment":{"id":"appconfig_delete_environment","name":"AppConfig Delete Environment","description":"Delete an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to delete"}}},"appconfig_delete_hosted_configuration_version":{"id":"appconfig_delete_hosted_configuration_version","name":"AppConfig Delete Hosted Configuration Version","description":"Delete a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID that owns the version"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to delete"}}},"appconfig_get_application":{"id":"appconfig_get_application","name":"AppConfig Get Application","description":"Get details about a single AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to retrieve"}}},"appconfig_get_configuration":{"id":"appconfig_get_configuration","name":"AppConfig Get Configuration","description":"Retrieve the latest deployed configuration for an AppConfig application, environment, and profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID or name to retrieve configuration for"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID or name to retrieve configuration for"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID or name to retrieve"}}},"appconfig_get_configuration_profile":{"id":"appconfig_get_configuration_profile","name":"AppConfig Get Configuration Profile","description":"Get details about a single AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to retrieve"}}},"appconfig_get_deployment":{"id":"appconfig_get_deployment","name":"AppConfig Get Deployment","description":"Get details about a specific AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment"}}},"appconfig_get_environment":{"id":"appconfig_get_environment","name":"AppConfig Get Environment","description":"Get details about a single AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to retrieve"}}},"appconfig_get_hosted_configuration_version":{"id":"appconfig_get_hosted_configuration_version","name":"AppConfig Get Hosted Configuration Version","description":"Retrieve a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to read the version from"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to retrieve"}}},"appconfig_list_applications":{"id":"appconfig_list_applications","name":"AppConfig List Applications","description":"List applications in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of applications to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_list_configuration_profiles":{"id":"appconfig_list_configuration_profiles","name":"AppConfig List Configuration Profiles","description":"List configuration profiles for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profiles"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of configuration profiles to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_list_deployment_strategies":{"id":"appconfig_list_deployment_strategies","name":"AppConfig List Deployment Strategies","description":"List deployment strategies available in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployment strategies to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_list_deployments":{"id":"appconfig_list_deployments","name":"AppConfig List Deployments","description":"List deployments for an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployments"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_list_environments":{"id":"appconfig_list_environments","name":"AppConfig List Environments","description":"List environments for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of environments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_list_hosted_configuration_versions":{"id":"appconfig_list_hosted_configuration_versions","name":"AppConfig List Hosted Configuration Versions","description":"List hosted configuration versions for an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to list versions for"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of versions to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}}},"appconfig_start_deployment":{"id":"appconfig_start_deployment","name":"AppConfig Start Deployment","description":"Start deploying a configuration version to an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to deploy in"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to deploy to"},"deploymentStrategyId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The deployment strategy ID to use"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to deploy"},"configurationVersion":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration version to deploy"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the deployment"}}},"appconfig_stop_deployment":{"id":"appconfig_stop_deployment","name":"AppConfig Stop Deployment","description":"Stop an in-progress AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment to stop"}}},"appconfig_update_application":{"id":"appconfig_update_application","name":"AppConfig Update Application","description":"Update the name or description of an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the application"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the application"}}},"appconfig_update_configuration_profile":{"id":"appconfig_update_configuration_profile","name":"AppConfig Update Configuration Profile","description":"Update the name, description, or retrieval role of an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the configuration profile"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"New ARN of the IAM role used to retrieve the configuration"}}},"appconfig_update_environment":{"id":"appconfig_update_environment","name":"AppConfig Update Environment","description":"Update the name or description of an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the environment"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the environment"}}},"arxiv_get_author_papers":{"id":"arxiv_get_author_papers","name":"ArXiv Get Author Papers","description":"Search for papers by a specific author on ArXiv.","version":"1.0.0","params":{"authorName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Author name to search for"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"}}},"arxiv_get_paper":{"id":"arxiv_get_paper","name":"ArXiv Get Paper","description":"Get detailed information about a specific ArXiv paper by its ID.","version":"1.0.0","params":{"paperId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ArXiv paper ID (e.g., \\"1706.03762\\")"}}},"arxiv_search":{"id":"arxiv_search","name":"ArXiv Search","description":"Search for academic papers on ArXiv by keywords, authors, titles, or other fields.","version":"1.0.0","params":{"searchQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query to execute"},"searchField":{"type":"string","required":false,"visibility":"user-only","description":"Field to search in: all, ti (title), au (author), abs (abstract), co (comment), jr (journal), cat (category), rn (report number)"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"},"sortBy":{"type":"string","required":false,"visibility":"user-only","description":"Sort by: relevance, lastUpdatedDate, submittedDate (default: relevance)"},"sortOrder":{"type":"string","required":false,"visibility":"user-only","description":"Sort order: ascending, descending (default: descending)"}}},"asana_add_comment":{"id":"asana_add_comment","name":"Asana Add Comment","description":"Add a comment (story) to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string)"},"text":{"type":"string","required":true,"visibility":"user-or-llm","description":"The text content of the comment"}},"oauth":{"required":true,"provider":"asana"}},"asana_add_followers":{"id":"asana_add_followers","name":"Asana Add Followers","description":"Add one or more followers to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task (numeric string)"},"followers":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of user GIDs to add as followers to the task"}},"oauth":{"required":true,"provider":"asana"}},"asana_create_project":{"id":"asana_create_project","name":"Asana Create Project","description":"Create a new project in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the project will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the project"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the project"}},"oauth":{"required":true,"provider":"asana"}},"asana_create_section":{"id":"asana_create_section","name":"Asana Create Section","description":"Create a new section in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to add the section to"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the section"}},"oauth":{"required":true,"provider":"asana"}},"asana_create_subtask":{"id":"asana_create_subtask","name":"Asana Create Subtask","description":"Create a subtask under an existing Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the parent Asana task (numeric string)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the subtask"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the subtask"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the subtask to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"}},"asana_create_task":{"id":"asana_create_task","name":"Asana Create Task","description":"Create a new task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the task will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the task to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"}},"asana_delete_task":{"id":"asana_delete_task","name":"Asana Delete Task","description":"Delete an Asana task by its GID (moves it to the trash)","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task to delete (numeric string)"}},"oauth":{"required":true,"provider":"asana"}},"asana_get_project":{"id":"asana_get_project","name":"Asana Get Project","description":"Retrieve a single Asana project by its GID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to retrieve"}},"oauth":{"required":true,"provider":"asana"}},"asana_get_projects":{"id":"asana_get_projects","name":"Asana Get Projects","description":"Retrieve all projects from an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to retrieve projects from"}},"oauth":{"required":true,"provider":"asana"}},"asana_get_task":{"id":"asana_get_task","name":"Asana Get Task","description":"Retrieve a single task by GID or get multiple tasks with filters","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":false,"visibility":"user-or-llm","description":"The globally unique identifier (GID) of the task. If not provided, will get multiple tasks."},"workspace":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to filter tasks (required when not using taskGid)"},"project":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to filter tasks"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of tasks to return (default: 50)"}},"oauth":{"required":true,"provider":"asana"}},"asana_list_sections":{"id":"asana_list_sections","name":"Asana List Sections","description":"List all sections in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to list sections from"}},"oauth":{"required":true,"provider":"asana"}},"asana_list_workspaces":{"id":"asana_list_workspaces","name":"Asana List Workspaces","description":"List all Asana workspaces and organizations the authenticated user belongs to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"}},"oauth":{"required":true,"provider":"asana"}},"asana_search_tasks":{"id":"asana_search_tasks","name":"Asana Search Tasks","description":"Search for tasks in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to search tasks in"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Text to search for in task names"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter tasks by assignee user GID"},"projects":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of Asana project GIDs (numeric strings) to filter tasks by"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Filter by completion status"}},"oauth":{"required":true,"provider":"asana"}},"asana_update_task":{"id":"asana_update_task","name":"Asana Update Task","description":"Update an existing task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string) of the task to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated name for the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated assignee user GID"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Mark task as completed or not completed"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"}},"ashby_add_candidate_tag":{"id":"ashby_add_candidate_tag","name":"Ashby Add Candidate Tag","description":"Adds a tag to a candidate in Ashby and returns the updated candidate.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the tag to"},"tagId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the tag to add"}}},"ashby_anonymize_candidate":{"id":"ashby_anonymize_candidate","name":"Ashby Anonymize Candidate","description":"Strips personally identifiable information from a candidate in Ashby. This does not delete the candidate - the record and its applications remain, with the PII removed. Ashby exposes no candidate deletion endpoint; true deletion is UI-only, restricted by role, and limited to a 10-day window. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the candidate to anonymize"}}},"ashby_change_application_source":{"id":"ashby_change_application_source","name":"Ashby Change Application Source","description":"Changes the source attributed to an existing application, so programmatically created applications report correctly on the recruiting side. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the application whose source should change"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the application to, as returned by List Sources. Omit only when unsetSource is true."},"unsetSource":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Set true to deliberately clear the application source. Required to unset, so that a missing or empty sourceId cannot wipe attribution by accident."}}},"ashby_change_application_stage":{"id":"ashby_change_application_stage","name":"Ashby Change Application Stage","description":"Moves an application to a different interview stage. Requires an archive reason when moving to an Archived stage.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the application to update the stage of"},"interviewStageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the interview stage to move the application to"},"archiveReasonId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Archive reason UUID. Required when moving to an Archived stage, ignored otherwise"}}},"ashby_create_application":{"id":"ashby_create_application","name":"Ashby Create Application","description":"Creates a new application for a candidate on a job. Optionally specify interview plan, stage, source, and credited user.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to consider for the job"},"jobId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the job to consider the candidate for"},"interviewPlanId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview plan to use (defaults to the job default plan)"},"interviewStageId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview stage to place the application in (defaults to first Lead stage)"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to set on the application"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the user the application is credited to"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to set as the application creation date (defaults to now)"}}},"ashby_create_candidate":{"id":"ashby_create_candidate","name":"Ashby Create Candidate","description":"Creates a new candidate record in Ashby.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"The candidate full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary email address for the candidate"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the candidate"},"linkedInUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"githubUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"GitHub profile URL"},"website":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal website URL"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the candidate to"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the Ashby user to credit with sourcing this candidate"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"Backdated creation timestamp in ISO 8601 (e.g. 2024-01-01T00:00:00Z). Defaults to now."},"alternateEmailAddresses":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of additional email address strings to add to the candidate, e.g. [\\"a@x.com\\",\\"b@y.com\\"]"}}},"ashby_create_note":{"id":"ashby_create_note","name":"Ashby Create Note","description":"Creates a note on a candidate in Ashby. Supports plain text and HTML content (bold, italic, underline, links, lists, code).","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the note to"},"note":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note content. If noteType is text/html, supports: , , , ,