Skip to content
18 changes: 13 additions & 5 deletions apps/sim/app/api/table/[tableId]/import/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import {
validateMapping,
wouldExceedRowLimit,
} from '@/lib/table'
import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
import { importAppendRows, importReplaceRows } from '@/lib/table/import-data'
import { getUserSettings } from '@/lib/users/queries'
import {
Expand Down Expand Up @@ -176,19 +177,26 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro
timezone = timezoneValidation.data
}

const delimiter = extensionValidation.data === 'tsv' ? '\t' : ','
const parser = createCsvParser(delimiter)
// The extension only picks the fallback — the separator is sniffed from the file's
// head so semicolon/pipe exports (European-locale Excel) don't land in one column.
const { delimiter, stream: csvStream } = await sniffCsvDelimiterFromStream(
file.stream,
extensionValidation.data === 'tsv' ? '\t' : ','
)
let headers: string[] = []
const parser = createCsvParser(delimiter, (parsedHeaders) => {
headers = parsedHeaders
})
// `.pipe` doesn't forward source errors; forward them so the iterator throws.
file.stream.on('error', (streamErr) => parser.destroy(streamErr))
file.stream.pipe(parser)
csvStream.on('error', (streamErr) => parser.destroy(streamErr))
csvStream.pipe(parser)
const rows: Record<string, unknown>[] = []
for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
rows.push(record)
}
if (rows.length === 0) {
return NextResponse.json({ error: 'CSV file has no data rows' }, { status: 400 })
}
const headers = Object.keys(rows[0])

let effectiveMapping = mapping ?? buildAutoMapping(headers, table.schema)
let prospectiveTable: TableDefinition = table
Expand Down
19 changes: 14 additions & 5 deletions apps/sim/app/api/table/import-csv/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
type TableDefinition,
type TableSchema,
} from '@/lib/table'
import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
import { getUserSettings } from '@/lib/users/queries'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
import {
Expand Down Expand Up @@ -107,12 +108,20 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
{ status: 400 }
)
}
const delimiter = extensionResult.data === 'tsv' ? '\t' : ','
// The extension only picks the fallback — the separator is sniffed from the file's
// head so semicolon/pipe exports (European-locale Excel) don't land in one column.
const { delimiter, stream: csvStream } = await sniffCsvDelimiterFromStream(
file.stream,
extensionResult.data === 'tsv' ? '\t' : ','
)

const parser = createCsvParser(delimiter)
let csvHeaders: string[] = []
const parser = createCsvParser(delimiter, (headers) => {
csvHeaders = headers
})
// `.pipe` doesn't forward source errors; forward them so the iterator throws.
file.stream.on('error', (err) => parser.destroy(err))
file.stream.pipe(parser)
csvStream.on('error', (err) => parser.destroy(err))
csvStream.pipe(parser)

interface ImportState {
table: TableDefinition
Expand All @@ -139,7 +148,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {

/** Infer the schema from the buffered sample and create the (empty) table. */
const buildTable = async (sampleRows: Record<string, unknown>[]): Promise<ImportState> => {
const inferred = inferSchemaFromCsv(Object.keys(sampleRows[0]), sampleRows)
const inferred = inferSchemaFromCsv(csvHeaders, sampleRows)
const schema: TableSchema = { columns: inferred.columns.map(normalizeColumn) }
const planLimits = await getWorkspaceTableLimits(workspaceId)
const tableName = sanitizeName(file.filename.replace(/\.[^.]+$/, ''), 'imported_table').slice(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@ import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { truncate } from '@sim/utils/string'
import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
import {
buildAutoMapping,
CSV_DELIMITER_SNIFF_BYTES,
type CsvDelimiter,
detectCsvDelimiter,
parseCsvBuffer,
} from '@/lib/table/import'
import type { TableDefinition } from '@/lib/table/types'
import {
type CsvImportMode,
Expand Down Expand Up @@ -107,15 +113,27 @@ interface ParsedCsv {
sampleRows: Record<string, unknown>[]
}

/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
/**
* Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line.
*
* The separator is sniffed from the same leading bytes the server sniffs, so the mapping shown
* here always matches the columns the import will actually produce.
*/
async function parseCsvPreview(file: File, fallbackDelimiter: CsvDelimiter) {
const sliced = file.size > CSV_PREVIEW_BYTES
const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
let bytes = new Uint8Array(await blob.arrayBuffer())
if (sliced) {
const lastNewline = bytes.lastIndexOf(0x0a)
if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
}
// The sniff sample is the whole file only when nothing was sliced off and it fits the window;
// otherwise it's a truncated prefix whose last line may be partial.
const delimiter = await detectCsvDelimiter(
bytes.subarray(0, CSV_DELIMITER_SNIFF_BYTES),
fallbackDelimiter,
{ complete: !sliced && bytes.length <= CSV_DELIMITER_SNIFF_BYTES }
)
Comment thread
waleedlatif1 marked this conversation as resolved.
return parseCsvBuffer(bytes, delimiter)
}

Expand Down Expand Up @@ -180,8 +198,7 @@ export function ImportCsvDialog({
setParsing(true)
setParseError(null)
try {
const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
const { headers, rows } = await parseCsvPreview(file, delimiter)
const { headers, rows } = await parseCsvPreview(file, ext === 'tsv' ? '\t' : ',')
const autoMapping = buildAutoMapping(headers, table.schema)
setParsed({
file,
Expand Down
80 changes: 80 additions & 0 deletions apps/sim/lib/table/csv-delimiter-stream.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { Readable } from 'node:stream'
import {
CSV_DELIMITER_SNIFF_BYTES,
type CsvDelimiter,
detectCsvDelimiter,
} from '@/lib/table/import'

export interface SniffedCsvStream {
delimiter: CsvDelimiter
/**
* The full file contents, replayed from byte zero. Use this in place of the
* source stream — the source has already been partially consumed.
*/
stream: Readable
}

/**
* Reads the head of a CSV/TSV stream, sniffs its field separator, then returns a
* stream that replays the buffered head followed by the untouched remainder.
*
* Memory is bounded: it buffers only the source chunks needed to reach the
* {@link CSV_DELIMITER_SNIFF_BYTES} window (one chunk past it, at worst), and the
* detection sample it copies is capped at exactly that window regardless of how
* large a single upstream chunk is. Those buffered chunks are then replayed
* *by reference* — never re-copied — so a multi-GB import stays O(sniff window)
* plus the single in-flight chunk. {@link detectCsvDelimiter} drops any partial
* trailing line, so no newline trimming is needed here.
*/
export async function sniffCsvDelimiterFromStream(
source: Readable,
fallback: CsvDelimiter = ','
): Promise<SniffedCsvStream> {
const reader = source[Symbol.asyncIterator]()
const chunks: Buffer[] = []
let size = 0
let exhausted = false

// Read until the buffered size *exceeds* the window (not just reaches it) or the stream ends.
// The `<=` is deliberate: a file whose size is exactly the window must still trigger one more
// read so EOF is observed and `exhausted` becomes true — otherwise it would be misjudged as a
// truncated prefix and disagree with the buffered callers (which pass complete for that size).
while (size <= CSV_DELIMITER_SNIFF_BYTES) {
const next = await reader.next()
if (next.done) {
exhausted = true
break
}
const chunk = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value as Uint8Array)
chunks.push(chunk)
size += chunk.length
}

// Copy at most the sniff window for detection — `Buffer.concat`'s length arg truncates,
// so an oversized final chunk can't inflate this allocation past CSV_DELIMITER_SNIFF_BYTES.
const sample = Buffer.concat(chunks, Math.min(size, CSV_DELIMITER_SNIFF_BYTES))
// `exhausted` (the loop stopped on end-of-stream, never on exceeding the window) means the whole
// file fit in the window, so its last row must count even without a trailing newline. Otherwise
// the sample is a truncated prefix whose final line may be partial and should be dropped.
const delimiter = await detectCsvDelimiter(sample, fallback, { complete: exhausted })

const stream = Readable.from(
(async function* replay() {
// Replay the already-read chunks by reference (no re-copy), then drain the rest.
for (const chunk of chunks) yield chunk
if (exhausted) return
while (true) {
const next = await reader.next()
if (next.done) return
yield next.value
}
})()
)

// `Readable.from` closes the generator on destroy, which returns the source
// iterator — but an early destroy of the wrapper before the generator is
// pulled would otherwise leak the source's socket.
stream.on('close', () => source.destroy())

return { delimiter, stream }
}
17 changes: 13 additions & 4 deletions apps/sim/lib/table/import-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
} from '@/lib/table'
import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing'
import { withGeneratedColumnIds } from '@/lib/table/column-keys'
import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream'
import { appendTableEvent } from '@/lib/table/events'
import {
addImportColumns,
Expand Down Expand Up @@ -103,6 +104,11 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
// Stream the file rather than buffering it — a ~1M-row import must never be held in memory.
source = await downloadFileStream({ key: fileKey, context: 'workspace' })

// The kickoff route's extension-derived delimiter is only the fallback — the separator is
// sniffed from the file's head so semicolon/pipe exports don't collapse into one column.
const sniffed = await sniffCsvDelimiterFromStream(source, delimiter)
const csvStream = sniffed.stream

// Append must continue after the existing rows; create/replace start empty. Read once up
// front (the import is the table's sole writer) and assign contiguous positions / threaded
// order keys from it.
Expand All @@ -123,11 +129,14 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
},
})

const parser = createCsvParser(delimiter)
let csvHeaders: string[] = []
const parser = createCsvParser(sniffed.delimiter, (headers) => {
csvHeaders = headers
})
// `.pipe` doesn't forward source errors; forward so the iterator throws.
source.on('error', (err) => parser.destroy(err))
csvStream.on('error', (err) => parser.destroy(err))
byteCounter.on('error', (err) => parser.destroy(err))
source.pipe(byteCounter).pipe(parser)
csvStream.pipe(byteCounter).pipe(parser)

let schema: TableSchema | null = null
let headerToColumn: Map<string, string> | null = null
Expand All @@ -142,7 +151,7 @@ export async function runTableImport(payload: TableImportPayload): Promise<void>
* map onto the existing schema, optionally auto-creating `createColumns` first.
*/
const resolveSetup = async () => {
const headers = Object.keys(sample[0])
const headers = csvHeaders

if (mode === 'create') {
const inferred = inferSchemaFromCsv(headers, sample)
Expand Down
Loading
Loading