Skip to content

fix(brightdata): align integration with live API docs, add markdown/mode/error-report support #5470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
waleedlatif1 merged 2 commits into from
Jul 7, 2026
Merged

fix(brightdata): align integration with live API docs, add markdown/mode/error-report support #5470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6f308d6
fix(brightdata): align integration with live API docs, add markdown/m…
waleedlatif1 Jul 7, 2026
aef2da2
fix(brightdata): default mode field to empty so it's opt-in like othe…
waleedlatif1 Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 83 additions & 3 deletions apps/sim/blocks/blocks/brightdata.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,18 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
value: () => 'raw',
condition: { field: 'operation', value: 'scrape_url' },
},
{
id: 'dataFormat',
title: 'Convert To',
type: 'dropdown',
options: [
{ label: 'None', id: '' },
{ label: 'Markdown', id: 'markdown' },
],
value: () => '',
mode: 'advanced',
condition: { field: 'operation', value: 'scrape_url' },
},
{
id: 'country',
title: 'Country',
Expand Down Expand Up @@ -119,6 +131,26 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
placeholder:
'Describe what you are looking for (e.g., "find official pricing pages and change notes")',
condition: { field: 'operation', value: 'discover' },
wandConfig: {
enabled: true,
prompt:
'Generate a concise description of what the agent is trying to accomplish, to help rank web-discovery results by relevance (e.g., "find official pricing pages and recent change notes"). Return ONLY the intent description - no explanations, no extra text.',
placeholder: 'Describe what you are trying to accomplish...',
},
},
{
id: 'mode',
title: 'Search Mode',
type: 'dropdown',
options: [
{ label: 'Standard (Default)', id: '' },
{ label: 'Deep', id: 'deep' },
{ label: 'Fast', id: 'fast' },
{ label: 'Zero Ranking', id: 'zeroRanking' },
],
value: () => '',
mode: 'advanced',
condition: { field: 'operation', value: 'discover' },
},
Comment thread
waleedlatif1 marked this conversation as resolved.
{
id: 'includeContent',
Expand All @@ -133,7 +165,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
type: 'dropdown',
options: [
{ label: 'JSON', id: 'json' },
{ label: 'Markdown', id: 'markdown' },
{ label: 'Markdown', id: 'md' },
],
value: () => 'json',
mode: 'advanced',
Expand All @@ -154,6 +186,12 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
placeholder: '[{"url": "https://example.com/product"}]',
condition: { field: 'operation', value: 'sync_scrape' },
required: { field: 'operation', value: 'sync_scrape' },
wandConfig: {
enabled: true,
prompt:
'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.',
placeholder: 'Describe the URLs to scrape...',
},
},
{
id: 'syncFormat',
Expand All @@ -167,6 +205,13 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
value: () => 'json',
condition: { field: 'operation', value: 'sync_scrape' },
},
{
id: 'syncIncludeErrors',
title: 'Include Errors',
type: 'switch',
mode: 'advanced',
condition: { field: 'operation', value: 'sync_scrape' },
},
{
id: 'datasetId',
title: 'Dataset ID',
Expand All @@ -182,6 +227,12 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
placeholder: '[{"url": "https://example.com/product"}]',
condition: { field: 'operation', value: 'scrape_dataset' },
required: { field: 'operation', value: 'scrape_dataset' },
wandConfig: {
enabled: true,
prompt:
'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.',
placeholder: 'Describe the URLs to scrape...',
},
},
{
id: 'datasetFormat',
Expand All @@ -194,6 +245,13 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
value: () => 'json',
condition: { field: 'operation', value: 'scrape_dataset' },
},
{
id: 'datasetIncludeErrors',
title: 'Include Errors',
type: 'switch',
mode: 'advanced',
condition: { field: 'operation', value: 'scrape_dataset' },
},
{
id: 'snapshotId',
title: 'Snapshot ID',
Expand Down Expand Up @@ -251,6 +309,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
result.url = params.url
if (params.format) result.format = params.format
if (params.country) result.country = params.country
if (params.dataFormat) result.dataFormat = params.dataFormat
break

case 'serp_search':
Expand All @@ -265,6 +324,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
case 'discover':
result.query = params.discoverQuery
if (params.numResults) result.numResults = Number(params.numResults)
if (params.mode) result.mode = params.mode
if (params.intent) result.intent = params.intent
if (params.includeContent != null) result.includeContent = params.includeContent
if (params.contentFormat) result.format = params.contentFormat
Expand All @@ -276,12 +336,15 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
result.datasetId = params.syncDatasetId
result.urls = params.syncUrls
if (params.syncFormat) result.format = params.syncFormat
if (params.syncIncludeErrors != null) result.includeErrors = params.syncIncludeErrors
break

case 'scrape_dataset':
result.datasetId = params.datasetId
result.urls = params.urls
if (params.datasetFormat) result.format = params.datasetFormat
if (params.datasetIncludeErrors != null)
result.includeErrors = params.datasetIncludeErrors
break

case 'snapshot_status':
Expand All @@ -308,33 +371,50 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
zone: { type: 'string', description: 'Bright Data zone name' },
url: { type: 'string', description: 'URL to scrape' },
format: { type: 'string', description: 'Response format' },
dataFormat: { type: 'string', description: 'Convert scraped content to markdown' },
country: { type: 'string', description: 'Country code for geo-targeting' },
query: { type: 'string', description: 'Search query' },
searchEngine: { type: 'string', description: 'Search engine to use' },
language: { type: 'string', description: 'Language code' },
numResults: { type: 'number', description: 'Number of results' },
discoverQuery: { type: 'string', description: 'Discover search query' },
intent: { type: 'string', description: 'Intent for ranking results' },
mode: { type: 'string', description: 'Search depth and ranking mode for discover' },
includeContent: { type: 'boolean', description: 'Include page content in discover results' },
contentFormat: { type: 'string', description: 'Content format for discover results' },
syncDatasetId: { type: 'string', description: 'Dataset scraper ID for sync scrape' },
syncUrls: { type: 'string', description: 'JSON array of URL objects for sync scrape' },
syncFormat: { type: 'string', description: 'Output format for sync scrape' },
syncIncludeErrors: {
type: 'boolean',
description: 'Include error reports in sync scrape results',
},
datasetId: { type: 'string', description: 'Dataset scraper ID' },
urls: { type: 'string', description: 'JSON array of URL objects to scrape' },
datasetFormat: { type: 'string', description: 'Dataset output format' },
datasetIncludeErrors: {
type: 'boolean',
description: 'Include error reports in scrape dataset results',
},
snapshotId: { type: 'string', description: 'Snapshot ID for status/download/cancel' },
downloadFormat: { type: 'string', description: 'Download output format' },
},
outputs: {
content: { type: 'string', description: 'Scraped page content' },
url: { type: 'string', description: 'URL that was scraped' },
statusCode: { type: 'number', description: 'HTTP status code' },
results: { type: 'json', description: 'Search or discover results array' },
results: {
type: 'json',
description:
'Search or discover results array: [{title, url, description, rank}] for SERP search, or [{url, title, description, relevanceScore, content}] for discover',
},
query: { type: 'string', description: 'Search query executed' },
searchEngine: { type: 'string', description: 'Search engine used' },
totalResults: { type: 'number', description: 'Total number of discover results' },
data: { type: 'json', description: 'Scraped data records' },
data: {
type: 'json',
description: 'Array of scraped result records with dataset-specific fields',
},
snapshotId: { type: 'string', description: 'Snapshot ID' },
isAsync: { type: 'boolean', description: 'Whether sync scrape fell back to async' },
status: { type: 'string', description: 'Job status' },
Expand Down
14 changes: 11 additions & 3 deletions apps/sim/tools/brightdata/discover.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export const brightDataDiscoverTool: ToolConfig<
id: 'brightdata_discover',
name: 'Bright Data Discover',
description:
'AI-powered web discovery that finds and ranks results by intent. Returns up to 1,000 results with optional cleaned page content for RAG and verification.',
'AI-powered web discovery that finds and ranks results by intent. Returns up to 20 results with optional cleaned page content for RAG and verification.',
version: '1.0.0',

params: {
Expand All @@ -37,7 +37,14 @@ export const brightDataDiscoverTool: ToolConfig<
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Number of results to return, up to 1000. Defaults to 10',
description: 'Number of results to return (1-20). Defaults to 10',
},
mode: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description:
'Search depth and ranking mode: "standard" (balanced), "deep" (exhaustive, broader search), "fast" (optimized for speed), or "zeroRanking" (raw volume without AI filtering). Defaults to "standard"',
},
intent: {
type: 'string',
Expand All @@ -56,7 +63,7 @@ export const brightDataDiscoverTool: ToolConfig<
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Response format: "json" or "markdown". Defaults to "json"',
description: 'Response format: "json" or "md". Defaults to "json"',
},
language: {
type: 'string',
Expand Down Expand Up @@ -84,6 +91,7 @@ export const brightDataDiscoverTool: ToolConfig<
query: params.query,
}
if (params.numResults) body.num_results = params.numResults
if (params.mode) body.mode = params.mode
if (params.intent) body.intent = params.intent
if (params.includeContent != null) body.include_content = params.includeContent
if (params.format) body.format = params.format
Expand Down
9 changes: 8 additions & 1 deletion apps/sim/tools/brightdata/scrape_dataset.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,21 @@ export const brightDataScrapeDatasetTool: ToolConfig<
visibility: 'user-or-llm',
description: 'Output format: "json" or "csv". Defaults to "json"',
},
includeErrors: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Whether to include a per-input error report in the results',
},
},

request: {
method: 'POST',
url: (params) => {
const queryParams = new URLSearchParams()
queryParams.set('dataset_id', params.datasetId)
queryParams.set('dataset_id', params.datasetId.trim())
queryParams.set('format', params.format || 'json')
if (params.includeErrors) queryParams.set('include_errors', 'true')
return `https://api.brightdata.com/datasets/v3/trigger?${queryParams.toString()}`
},
headers: (params) => ({
Expand Down
8 changes: 8 additions & 0 deletions apps/sim/tools/brightdata/scrape_url.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@ export const brightDataScrapeUrlTool: ToolConfig<
visibility: 'user-or-llm',
description: 'Two-letter country code for geo-targeting (e.g., "us", "gb")',
},
dataFormat: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description:
'Convert the response to "markdown" instead of raw HTML, useful for feeding page content to an LLM. Omit for the default HTML/JSON response',
},
},

request: {
Expand All @@ -62,6 +69,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
format: params.format || 'raw',
}
if (params.country) body.country = params.country
if (params.dataFormat) body.data_format = params.dataFormat
return body
},
},
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/tools/brightdata/sync_scrape.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export const brightDataSyncScrapeTool: ToolConfig<
method: 'POST',
url: (params) => {
const queryParams = new URLSearchParams()
queryParams.set('dataset_id', params.datasetId)
queryParams.set('dataset_id', params.datasetId.trim())
queryParams.set('format', params.format || 'json')
if (params.includeErrors) queryParams.set('include_errors', 'true')
return `https://api.brightdata.com/datasets/v3/scrape?${queryParams.toString()}`
Expand Down
3 changes: 3 additions & 0 deletions apps/sim/tools/brightdata/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export interface BrightDataScrapeUrlParams {
url: string
format?: string
country?: string
dataFormat?: string
}

export interface BrightDataScrapeUrlResponse extends ToolResponse {
Expand Down Expand Up @@ -44,6 +45,7 @@ export interface BrightDataScrapeDatasetParams {
datasetId: string
urls: string
format?: string
includeErrors?: boolean
}

export interface BrightDataScrapeDatasetResponse extends ToolResponse {
Expand Down Expand Up @@ -113,6 +115,7 @@ export interface BrightDataDiscoverParams {
apiKey: string
query: string
numResults?: number
mode?: string
intent?: string
includeContent?: boolean
format?: string
Expand Down
Loading