-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Expand file tree
/
Copy pathextract.ts
More file actions
138 lines (132 loc) · 4.72 KB
/
Copy pathextract.ts
File metadata and controls
138 lines (132 loc) · 4.72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
import { contextDevHosting } from '@/tools/context_dev/hosting'
import type { ContextDevExtractParams, ContextDevExtractResponse } from '@/tools/context_dev/types'
import {
CONTEXT_DEV_BASE_URL,
CREDIT_OUTPUTS,
contextDevJsonHeaders,
extractCreditMetadata,
parseContextDevResponse,
} from '@/tools/context_dev/utils'
import type { ToolConfig } from '@/tools/types'
export const contextDevExtractTool: ToolConfig<ContextDevExtractParams, ContextDevExtractResponse> =
{
id: 'context_dev_extract',
name: 'Context.dev Extract',
description: 'Crawl a website and extract structured data matching a provided JSON schema.',
version: '1.0.0',
hosting: contextDevHosting<ContextDevExtractParams>(),
params: {
url: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The starting website URL (must include http:// or https://)',
},
schema: {
type: 'json',
required: true,
visibility: 'user-or-llm',
description: 'JSON Schema describing the structure of the data to extract',
},
instructions: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Optional extraction guidance for link prioritization (max 2000 chars)',
},
factCheck: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Require extracted values to be grounded in page facts (default: false)',
},
followSubdomains: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Follow links on subdomains of the starting domain (default: false)',
},
maxPages: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Maximum number of pages to analyze (1-50, default: 5)',
},
maxDepth: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Maximum link depth from the starting URL',
},
maxAgeMs: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Cache duration in milliseconds (0-2592000000, default: 604800000)',
},
stopAfterMs: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Soft crawl time budget in milliseconds (10000-110000, default: 80000)',
},
timeoutMS: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Request timeout in milliseconds (1000-300000)',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Context.dev API key',
},
},
request: {
method: 'POST',
url: () => `${CONTEXT_DEV_BASE_URL}/web/extract`,
headers: (params) => contextDevJsonHeaders(params.apiKey),
body: (params) => {
const body: Record<string, any> = { url: params.url, schema: params.schema }
if (params.instructions) body.instructions = params.instructions
if (params.factCheck != null) body.factCheck = params.factCheck
if (params.followSubdomains != null) body.followSubdomains = params.followSubdomains
if (params.maxPages != null) body.maxPages = params.maxPages
if (params.maxDepth != null) body.maxDepth = params.maxDepth
if (params.maxAgeMs != null) body.maxAgeMs = params.maxAgeMs
if (params.stopAfterMs != null) body.stopAfterMs = params.stopAfterMs
if (params.timeoutMS != null) body.timeoutMS = params.timeoutMS
return body
},
},
transformResponse: async (response: Response) => {
const data = await parseContextDevResponse(response)
return {
success: true,
output: {
status: data.status ?? '',
url: data.url ?? '',
urlsAnalyzed: data.urls_analyzed ?? [],
data: data.data ?? {},
metadata: data.metadata ?? {},
...extractCreditMetadata(data.key_metadata),
},
}
},
outputs: {
status: { type: 'string', description: 'Extraction status' },
url: { type: 'string', description: 'The starting URL that was crawled' },
urlsAnalyzed: {
type: 'array',
description: 'URLs that were analyzed during extraction',
items: { type: 'string', description: 'Analyzed page URL' },
},
data: { type: 'json', description: 'Structured data matching the requested schema' },
metadata: {
type: 'object',
description: 'Crawl summary (numUrls, maxCrawlDepth, numSucceeded, numFailed, numSkipped)',
},
...CREDIT_OUTPUTS,
},
}