Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,26 @@ import {
TableRow,
toast,
} from '@/components/emcn'
import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
import type { TableDefinition } from '@/lib/table/types'
import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables'
import {
type CsvImportMode,
cancelTableImport,
useImportCsvIntoTable,
useImportCsvIntoTableAsync,
} from '@/hooks/queries/tables'
import { useImportTrayStore } from '@/stores/table/import-tray/store'

const logger = createLogger('ImportCsvDialog')

const MAX_SAMPLE_ROWS = 5
const MAX_EXAMPLES_IN_ERROR = 3
/**
* Bytes read for the preview/mapping. We never parse the whole file client-side — the importer
* streams it server-side and the DB row-count trigger enforces the row limit.
*/
const CSV_PREVIEW_BYTES = 512 * 1024
/**
* Sentinel value for the "Do not import" option in the mapping combobox. The
* whitespace is intentional: valid column names must match `NAME_PATTERN`
Expand Down Expand Up @@ -92,7 +104,18 @@ interface ParsedCsv {
file: File
headers: string[]
sampleRows: Record<string, unknown>[]
totalRows: number
}

/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
const sliced = file.size > CSV_PREVIEW_BYTES
const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
let bytes = new Uint8Array(await blob.arrayBuffer())
if (sliced) {
const lastNewline = bytes.lastIndexOf(0x0a)
if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
}
return parseCsvBuffer(bytes, delimiter)
}

export function ImportCsvDialog({
Expand All @@ -110,6 +133,7 @@ export function ImportCsvDialog({
const [createHeaders, setCreateHeaders] = useState<Set<string>>(new Set())
const [mode, setMode] = useState<CsvImportMode>('append')
const importMutation = useImportCsvIntoTable()
const importAsyncMutation = useImportCsvIntoTableAsync()

function resetState() {
setParsed(null)
Expand Down Expand Up @@ -155,15 +179,13 @@ export function ImportCsvDialog({
setParsing(true)
setParseError(null)
try {
const arrayBuffer = await file.arrayBuffer()
const delimiter = ext === 'tsv' ? '\t' : ','
const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter)
const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
const { headers, rows } = await parseCsvPreview(file, delimiter)
const autoMapping = buildAutoMapping(headers, table.schema)
setParsed({
file,
headers,
sampleRows: rows.slice(0, MAX_SAMPLE_ROWS),
totalRows: rows.length,
})
setMapping(autoMapping)
} catch (err) {
Expand Down Expand Up @@ -256,36 +278,70 @@ export function ImportCsvDialog({
}
}, [mapping, parsed?.headers, table.schema.columns, createHeaders])

const appendCapacityDeficit =
parsed && mode === 'append' && table.rowCount + parsed.totalRows > table.maxRows
? table.rowCount + parsed.totalRows - table.maxRows
: 0

const replaceCapacityDeficit =
parsed && mode === 'replace' && parsed.totalRows > table.maxRows
? parsed.totalRows - table.maxRows
: 0

const canSubmit =
parsed !== null &&
!importMutation.isPending &&
!importAsyncMutation.isPending &&
missingRequired.length === 0 &&
duplicateTargets.length === 0 &&
mappedCount + createCount > 0 &&
appendCapacityDeficit === 0 &&
replaceCapacityDeficit === 0
mappedCount + createCount > 0

async function handleSubmit() {
if (!parsed || !canSubmit) return
setSubmitError(null)
const createColumns = createHeaders.size > 0 ? [...createHeaders] : undefined

// Large files can't be POSTed through the server (request-body cap) — upload them
// straight to storage and import in the background instead. Seed the header tray and
// close the dialog immediately so the indicator is visible during the upload, then run
// the upload + kickoff in the background (don't block the dialog on it).
if (parsed.file.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES) {
useImportTrayStore.getState().startUpload({
uploadId: table.id,
workspaceId,
title: parsed.file.name,
})
onOpenChange(false)
toast.success(`Importing "${parsed.file.name}" into "${table.name}" in the background`)
importAsyncMutation.mutate(
{
workspaceId,
tableId: table.id,
file: parsed.file,
mode,
mapping,
createColumns,
onProgress: (percent) => {
useImportTrayStore.getState().setUploadPercent(table.id, percent)
},
},
{
onSuccess: (data) => {
useImportTrayStore.getState().endUpload(table.id)
// The server row drives the tray once the list refetches. If canceled mid-upload, flag
// the id so it's not shown and cancel the worker server-side.
if (useImportTrayStore.getState().consumeCanceled(table.id) && data?.importId) {
useImportTrayStore.getState().cancel(table.id)
void cancelTableImport(workspaceId, table.id, data.importId).catch(() => {})
}
},
onError: () => {
// The hook's onError surfaces the toast; just clear the tray indicator here.
useImportTrayStore.getState().endUpload(table.id)
},
}
)
return
}

try {
const result = await importMutation.mutateAsync({
workspaceId,
tableId: table.id,
file: parsed.file,
mode,
mapping,
createColumns: createHeaders.size > 0 ? [...createHeaders] : undefined,
createColumns,
})
const data = result.data
if (mode === 'append') {
Expand All @@ -307,11 +363,7 @@ export function ImportCsvDialog({
}
}

const hasWarning =
missingRequired.length > 0 ||
duplicateTargets.length > 0 ||
appendCapacityDeficit > 0 ||
replaceCapacityDeficit > 0
const hasWarning = missingRequired.length > 0 || duplicateTargets.length > 0

return (
<ChipModal
Expand Down Expand Up @@ -344,7 +396,7 @@ export function ImportCsvDialog({
{parsed.file.name}
</span>
<span className='text-[var(--text-tertiary)] text-xs'>
{parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns
{parsed.headers.length} columns
</span>
</div>
<Button variant='ghost' size='sm' onClick={resetState}>
Expand Down Expand Up @@ -442,20 +494,6 @@ export function ImportCsvDialog({
Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one)
</p>
)}
{appendCapacityDeficit > 0 && (
<p className='text-[var(--text-error)] text-caption leading-tight'>
Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '}
{appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to
Replace.
</p>
)}
{replaceCapacityDeficit > 0 && (
<p className='text-[var(--text-error)] text-caption leading-tight'>
CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table limit
of {table.maxRows.toLocaleString()} by {replaceCapacityDeficit.toLocaleString()}
.
</p>
)}
</div>
)}

Expand Down
85 changes: 68 additions & 17 deletions apps/sim/app/workspace/[workspaceId]/tables/tables.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { createLogger } from '@sim/logger'
import { generateId } from '@sim/utils/id'
import { useParams, useRouter } from 'next/navigation'
import type { ComboboxOption } from '@/components/emcn'
import { ChipCombobox, ChipConfirmModal, toast, Upload } from '@/components/emcn'
import { Columns3, Rows3, Table as TableIcon } from '@/components/emcn/icons'
import type { TableDefinition } from '@/lib/table'
import { generateUniqueTableName } from '@/lib/table/constants'
import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES, generateUniqueTableName } from '@/lib/table/constants'
import type {
FilterTag,
ResourceColumn,
Expand All @@ -24,14 +25,17 @@ import {
import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
import {
ImportCsvDialog,
ImportProgressMenu,
TablesListContextMenu,
} from '@/app/workspace/[workspaceId]/tables/components'
import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu'
import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
import {
cancelTableImport,
downloadTableExport,
useCreateTable,
useDeleteTable,
useImportCsvAsync,
useRenameTable,
useTablesList,
useUploadCsvToTable,
Expand All @@ -40,6 +44,7 @@ import { useWorkspaceMembersQuery } from '@/hooks/queries/workspace'
import { useDebounce } from '@/hooks/use-debounce'
import { useInlineRename } from '@/hooks/use-inline-rename'
import { usePermissionConfig } from '@/hooks/use-permission-config'
import { useImportTrayStore } from '@/stores/table/import-tray/store'

const logger = createLogger('Tables')

Expand Down Expand Up @@ -76,6 +81,7 @@ export function Tables() {
const renameTable = useRenameTable(workspaceId)
const createTable = useCreateTable(workspaceId)
const uploadCsv = useUploadCsvToTable()
const importCsvAsync = useImportCsvAsync()

const tableRename = useInlineRename({
onSave: (tableId, name) => renameTable.mutate({ tableId, name }),
Expand Down Expand Up @@ -407,37 +413,80 @@ export function Tables() {
const list = e.target.files
if (!list || list.length === 0 || !workspaceId) return

try {
setUploading(true)
const csvFiles = Array.from(list).filter((f) => {
const ext = f.name.split('.').pop()?.toLowerCase()
return ext === 'csv' || ext === 'tsv'
})

if (csvFiles.length === 0) {
toast.error('No CSV or TSV files selected')
if (csvInputRef.current) csvInputRef.current.value = ''
return
}

const csvFiles = Array.from(list).filter((f) => {
const ext = f.name.split('.').pop()?.toLowerCase()
return ext === 'csv' || ext === 'tsv'
})
// Large files can't be POSTed through the server (request-body cap) — upload them
// straight to storage and import in the background. These are tracked by the import
// tray, never the header upload button, so don't touch uploading/uploadProgress here.
const asyncFiles = csvFiles.filter((f) => f.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
const syncFiles = csvFiles.filter((f) => f.size < CSV_ASYNC_IMPORT_THRESHOLD_BYTES)

if (csvFiles.length === 0) {
toast.error('No CSV or TSV files selected')
return
try {
for (const file of asyncFiles) {
// Show the indicator immediately under a temporary id (the real table id doesn't
// exist until kickoff returns), then let the tray track it. Don't redirect — the
// table is still empty/importing, so stay on the list.
const pendingId = `pending_${generateId()}`
useImportTrayStore
.getState()
.startUpload({ uploadId: pendingId, workspaceId, title: file.name })
toast.success(`Importing "${file.name}" in the background`)
try {
const result = await importCsvAsync.mutateAsync({
workspaceId,
file,
onProgress: (percent) => {
useImportTrayStore.getState().setUploadPercent(pendingId, percent)
},
})
useImportTrayStore.getState().endUpload(pendingId)
// The server row drives the tray once the list refetches (mutation invalidates it).
// If canceled mid-upload, flag the real id so it's not shown and cancel server-side.
if (
result?.tableId &&
result.importId &&
useImportTrayStore.getState().consumeCanceled(pendingId)
) {
useImportTrayStore.getState().cancel(result.tableId)
void cancelTableImport(workspaceId, result.tableId, result.importId).catch(() => {})
}
} catch {
// The hook's onError surfaces the toast; just clear the tray indicator here.
useImportTrayStore.getState().endUpload(pendingId)
}
Comment thread
TheodoreSpeaks marked this conversation as resolved.
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.

setUploadProgress({ completed: 0, total: csvFiles.length })
if (syncFiles.length === 0) return

setUploading(true)
setUploadProgress({ completed: 0, total: syncFiles.length })
const failed: string[] = []

for (let i = 0; i < csvFiles.length; i++) {
for (let i = 0; i < syncFiles.length; i++) {
const file = syncFiles[i]
try {
const result = await uploadCsv.mutateAsync({ workspaceId, file: csvFiles[i] })
const result = await uploadCsv.mutateAsync({ workspaceId, file })

if (csvFiles.length === 1) {
if (syncFiles.length === 1 && asyncFiles.length === 0) {
const tableId = result?.data?.table?.id
if (tableId) {
router.push(`/workspace/${workspaceId}/tables/${tableId}`)
}
}
} catch (err) {
failed.push(csvFiles[i].name)
failed.push(file.name)
logger.error('Error uploading CSV:', err)
} finally {
setUploadProgress({ completed: i + 1, total: csvFiles.length })
setUploadProgress({ completed: i + 1, total: syncFiles.length })
}
}

Expand All @@ -459,7 +508,8 @@ export function Tables() {
}
}
},
[workspaceId, router, uploadCsv]
// eslint-disable-next-line react-hooks/exhaustive-deps -- mutation objects are unstable; mutateAsync is stable in v5
[workspaceId, router]
)
Comment thread
TheodoreSpeaks marked this conversation as resolved.

const handleListUploadCsv = useCallback(() => {
Expand Down Expand Up @@ -508,6 +558,7 @@ export function Tables() {
sort={sortConfig}
filter={filterContent}
filterTags={filterTags}
leadingActions={<ImportProgressMenu workspaceId={workspaceId} />}
headerActions={[
{
label: uploadButtonLabel,
Expand Down
Loading