Skip to content

Commit 1f11d4f

Browse files
author
Frank
committed
zen: data dumper
1 parent b308503 commit 1f11d4f

File tree

27 files changed

+764
-717
lines changed

27 files changed

+764
-717
lines changed

‎github/sst-env.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@
66
/// <reference path="../sst-env.d.ts" />
77

88
import "sst"
9-
export {}
9+
export {}

‎infra/console.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ const gatewayKv = new sst.cloudflare.Kv("GatewayKv")
116116
// CONSOLE
117117
////////////////
118118

119-
const bucket = new sst.cloudflare.Bucket("ConsoleData")
119+
const oldBucket = new sst.cloudflare.Bucket("ConsoleData")
120+
const bucket = new sst.cloudflare.Bucket("ZenData")
120121

121122
const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID")
122123
const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY")

‎packages/console/app/src/routes/zen/util/dataDumper.ts‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,37 @@
11
import { Resource, waitUntil } from "@opencode-ai/console-resource"
22

3-
export function createDataDumper(sessionId: string, requestId: string) {
3+
export function createDataDumper(sessionId: string, requestId: string, projectId: string) {
44
if (Resource.App.stage !== "production") return
5+
if (sessionId === "") return
56

6-
let data: Record<string, any> = {}
7-
let modelName: string | undefined
7+
let data: Record<string, any> = { sessionId, requestId, projectId }
8+
let metadata: Record<string, any> = { sessionId, requestId, projectId }
89

910
return {
10-
provideModel: (model?: string) => (modelName = model),
11+
provideModel: (model?: string) => {
12+
data.modelName = model
13+
metadata.modelName = model
14+
},
1115
provideRequest: (request: string) => (data.request = request),
1216
provideResponse: (response: string) => (data.response = response),
1317
provideStream: (chunk: string) => (data.response = (data.response ?? "") + chunk),
1418
flush: () => {
15-
if (!modelName) return
19+
if (!data.modelName) return
20+
21+
const timestamp = new Date().toISOString().replace(/[^0-9]/g, "")
1622

17-
const str = new Date().toISOString().replace(/[^0-9]/g, "")
18-
const yyyymmdd = str.substring(0, 8)
19-
const hh = str.substring(8, 10)
23+
waitUntil(
24+
Resource.ZenData.put(
25+
`data/${data.modelName}/${sessionId}/${requestId}.json`,
26+
JSON.stringify({ timestamp, ...data }),
27+
),
28+
)
2029

2130
waitUntil(
22-
Resource.ConsoleData.put(`${yyyymmdd}/${hh}/${modelName}/${sessionId}/${requestId}.json`, JSON.stringify(data)),
31+
Resource.ZenData.put(
32+
`meta/${data.modelName}/${timestamp}/${requestId}.json`,
33+
JSON.stringify({ timestamp, ...metadata }),
34+
),
2335
)
2436
},
2537
}

‎packages/console/app/src/routes/zen/util/handler.ts‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,7 @@ import { ModelTable } from "@opencode-ai/console-core/schema/model.sql.js"
1313
import { ProviderTable } from "@opencode-ai/console-core/schema/provider.sql.js"
1414
import { logger } from "./logger"
1515
import { AuthError, CreditsError, MonthlyLimitError, UserLimitError, ModelError, RateLimitError } from "./error"
16-
import {
17-
createBodyConverter,
18-
createStreamPartConverter,
19-
createResponseConverter,
20-
ProviderHelper,
21-
UsageInfo,
22-
} from "./provider/provider"
16+
import { createBodyConverter, createStreamPartConverter, createResponseConverter, UsageInfo } from "./provider/provider"
2317
import { anthropicHelper } from "./provider/anthropic"
2418
import { googleHelper } from "./provider/google"
2519
import { openaiHelper } from "./provider/openai"
@@ -61,14 +55,15 @@ export async function handler(
6155
const ip = input.request.headers.get("x-real-ip") ?? ""
6256
const sessionId = input.request.headers.get("x-opencode-session") ?? ""
6357
const requestId = input.request.headers.get("x-opencode-request") ?? ""
58+
const projectId = input.request.headers.get("x-opencode-project") ?? ""
6459
logger.metric({
6560
is_tream: isStream,
6661
session: sessionId,
6762
request: requestId,
6863
})
6964
const zenData = ZenData.list()
7065
const modelInfo = validateModel(zenData, model)
71-
const dataDumper = createDataDumper(sessionId, requestId)
66+
const dataDumper = createDataDumper(sessionId, requestId, projectId)
7267
const trialLimiter = createTrialLimiter(modelInfo.trial?.limit, ip)
7368
const isTrial = await trialLimiter?.isTrial()
7469
const rateLimiter = createRateLimiter(modelInfo.id, modelInfo.rateLimit, ip)

‎packages/console/app/sst-env.d.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@
66
/// <reference path="../../../sst-env.d.ts" />
77

88
import "sst"
9-
export {}
9+
export {}

‎packages/console/core/sst-env.d.ts‎

Lines changed: 119 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -6,126 +6,131 @@
66
import "sst"
77
declare module "sst" {
88
export interface Resource {
9-
ADMIN_SECRET: {
10-
type: "sst.sst.Secret"
11-
value: string
12-
}
13-
AUTH_API_URL: {
14-
type: "sst.sst.Linkable"
15-
value: string
16-
}
17-
AWS_SES_ACCESS_KEY_ID: {
18-
type: "sst.sst.Secret"
19-
value: string
20-
}
21-
AWS_SES_SECRET_ACCESS_KEY: {
22-
type: "sst.sst.Secret"
23-
value: string
24-
}
25-
CLOUDFLARE_API_TOKEN: {
26-
type: "sst.sst.Secret"
27-
value: string
28-
}
29-
CLOUDFLARE_DEFAULT_ACCOUNT_ID: {
30-
type: "sst.sst.Secret"
31-
value: string
32-
}
33-
Console: {
34-
type: "sst.cloudflare.SolidStart"
35-
url: string
36-
}
37-
Database: {
38-
database: string
39-
host: string
40-
password: string
41-
port: number
42-
type: "sst.sst.Linkable"
43-
username: string
44-
}
45-
Desktop: {
46-
type: "sst.cloudflare.StaticSite"
47-
url: string
48-
}
49-
EMAILOCTOPUS_API_KEY: {
50-
type: "sst.sst.Secret"
51-
value: string
52-
}
53-
GITHUB_APP_ID: {
54-
type: "sst.sst.Secret"
55-
value: string
56-
}
57-
GITHUB_APP_PRIVATE_KEY: {
58-
type: "sst.sst.Secret"
59-
value: string
60-
}
61-
GITHUB_CLIENT_ID_CONSOLE: {
62-
type: "sst.sst.Secret"
63-
value: string
64-
}
65-
GITHUB_CLIENT_SECRET_CONSOLE: {
66-
type: "sst.sst.Secret"
67-
value: string
68-
}
69-
GOOGLE_CLIENT_ID: {
70-
type: "sst.sst.Secret"
71-
value: string
72-
}
73-
HONEYCOMB_API_KEY: {
74-
type: "sst.sst.Secret"
75-
value: string
76-
}
77-
R2AccessKey: {
78-
type: "sst.sst.Secret"
79-
value: string
80-
}
81-
R2SecretKey: {
82-
type: "sst.sst.Secret"
83-
value: string
84-
}
85-
STRIPE_SECRET_KEY: {
86-
type: "sst.sst.Secret"
87-
value: string
88-
}
89-
STRIPE_WEBHOOK_SECRET: {
90-
type: "sst.sst.Linkable"
91-
value: string
92-
}
93-
Web: {
94-
type: "sst.cloudflare.Astro"
95-
url: string
96-
}
97-
ZEN_MODELS1: {
98-
type: "sst.sst.Secret"
99-
value: string
100-
}
101-
ZEN_MODELS2: {
102-
type: "sst.sst.Secret"
103-
value: string
104-
}
105-
ZEN_MODELS3: {
106-
type: "sst.sst.Secret"
107-
value: string
108-
}
109-
ZEN_MODELS4: {
110-
type: "sst.sst.Secret"
111-
value: string
9+
"ADMIN_SECRET": {
10+
"type": "sst.sst.Secret"
11+
"value": string
12+
}
13+
"AUTH_API_URL": {
14+
"type": "sst.sst.Linkable"
15+
"value": string
16+
}
17+
"AWS_SES_ACCESS_KEY_ID": {
18+
"type": "sst.sst.Secret"
19+
"value": string
20+
}
21+
"AWS_SES_SECRET_ACCESS_KEY": {
22+
"type": "sst.sst.Secret"
23+
"value": string
24+
}
25+
"CLOUDFLARE_API_TOKEN": {
26+
"type": "sst.sst.Secret"
27+
"value": string
28+
}
29+
"CLOUDFLARE_DEFAULT_ACCOUNT_ID": {
30+
"type": "sst.sst.Secret"
31+
"value": string
32+
}
33+
"Console": {
34+
"type": "sst.cloudflare.SolidStart"
35+
"url": string
36+
}
37+
"Database": {
38+
"database": string
39+
"host": string
40+
"password": string
41+
"port": number
42+
"type": "sst.sst.Linkable"
43+
"username": string
44+
}
45+
"Desktop": {
46+
"type": "sst.cloudflare.StaticSite"
47+
"url": string
48+
}
49+
"EMAILOCTOPUS_API_KEY": {
50+
"type": "sst.sst.Secret"
51+
"value": string
52+
}
53+
"Enterprise": {
54+
"type": "sst.cloudflare.SolidStart"
55+
"url": string
56+
}
57+
"GITHUB_APP_ID": {
58+
"type": "sst.sst.Secret"
59+
"value": string
60+
}
61+
"GITHUB_APP_PRIVATE_KEY": {
62+
"type": "sst.sst.Secret"
63+
"value": string
64+
}
65+
"GITHUB_CLIENT_ID_CONSOLE": {
66+
"type": "sst.sst.Secret"
67+
"value": string
68+
}
69+
"GITHUB_CLIENT_SECRET_CONSOLE": {
70+
"type": "sst.sst.Secret"
71+
"value": string
72+
}
73+
"GOOGLE_CLIENT_ID": {
74+
"type": "sst.sst.Secret"
75+
"value": string
76+
}
77+
"HONEYCOMB_API_KEY": {
78+
"type": "sst.sst.Secret"
79+
"value": string
80+
}
81+
"R2AccessKey": {
82+
"type": "sst.sst.Secret"
83+
"value": string
84+
}
85+
"R2SecretKey": {
86+
"type": "sst.sst.Secret"
87+
"value": string
88+
}
89+
"STRIPE_SECRET_KEY": {
90+
"type": "sst.sst.Secret"
91+
"value": string
92+
}
93+
"STRIPE_WEBHOOK_SECRET": {
94+
"type": "sst.sst.Linkable"
95+
"value": string
96+
}
97+
"Web": {
98+
"type": "sst.cloudflare.Astro"
99+
"url": string
100+
}
101+
"ZEN_MODELS1": {
102+
"type": "sst.sst.Secret"
103+
"value": string
104+
}
105+
"ZEN_MODELS2": {
106+
"type": "sst.sst.Secret"
107+
"value": string
108+
}
109+
"ZEN_MODELS3": {
110+
"type": "sst.sst.Secret"
111+
"value": string
112+
}
113+
"ZEN_MODELS4": {
114+
"type": "sst.sst.Secret"
115+
"value": string
112116
}
113117
}
114118
}
115-
// cloudflare
116-
import * as cloudflare from "@cloudflare/workers-types"
119+
// cloudflare
120+
import * as cloudflare from "@cloudflare/workers-types";
117121
declare module "sst" {
118122
export interface Resource {
119-
Api: cloudflare.Service
120-
AuthApi: cloudflare.Service
121-
AuthStorage: cloudflare.KVNamespace
122-
Bucket: cloudflare.R2Bucket
123-
ConsoleData: cloudflare.R2Bucket
124-
EnterpriseStorage: cloudflare.R2Bucket
125-
GatewayKv: cloudflare.KVNamespace
126-
LogProcessor: cloudflare.Service
123+
"Api": cloudflare.Service
124+
"AuthApi": cloudflare.Service
125+
"AuthStorage": cloudflare.KVNamespace
126+
"Bucket": cloudflare.R2Bucket
127+
"ConsoleData": cloudflare.R2Bucket
128+
"EnterpriseStorage": cloudflare.R2Bucket
129+
"GatewayKv": cloudflare.KVNamespace
130+
"LogProcessor": cloudflare.Service
131+
"ZenData": cloudflare.R2Bucket
127132
}
128133
}
129134

130135
import "sst"
131-
export {}
136+
export {}

0 commit comments

Comments
 (0)