Skip to content

Commit fe8fa6b

Browse files
jamesbhobbsjamesbhobbs
committed
fix: auto fix dep management due to file read
1 parent e468bc6 commit fe8fa6b

1 file changed

Lines changed: 170 additions & 38 deletions

File tree

‎data-app/index.html‎

Lines changed: 170 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1242,6 +1242,66 @@ <h3>Run in Browser with WebAssembly</h3>
12421242
return json.dumps({"errorMessage": f"{e.__class__.__name__}: {str(e)}"})
12431243
`
12441244

1245+
// Topological sort using Kahn's algorithm
1246+
function topologicalSort(dag) {
1247+
if (!dag || !dag.nodes.length) return []
1248+
1249+
// Build adjacency list and in-degree map
1250+
const adjacency = new Map() // from -> [to]
1251+
const inDegree = new Map() // node -> count of incoming edges
1252+
const nodeOrder = new Map() // blockId -> order (for tie-breaking)
1253+
1254+
// Initialize all nodes
1255+
for (const node of dag.nodes) {
1256+
adjacency.set(node.blockId, [])
1257+
inDegree.set(node.blockId, 0)
1258+
nodeOrder.set(node.blockId, node.order ?? 0)
1259+
}
1260+
1261+
// Build edges (from defines to uses)
1262+
for (const edge of dag.edges) {
1263+
const fromId = edge.fromBlockId
1264+
const toId = edge.toBlockId
1265+
if (adjacency.has(fromId) && inDegree.has(toId)) {
1266+
adjacency.get(fromId).push(toId)
1267+
inDegree.set(toId, inDegree.get(toId) + 1)
1268+
}
1269+
}
1270+
1271+
// Find all nodes with no incoming edges (sorted by original order for consistency)
1272+
const queue = [...inDegree.entries()]
1273+
.filter(([_, deg]) => deg === 0)
1274+
.map(([id]) => id)
1275+
.sort((a, b) => (nodeOrder.get(a) ?? 0) - (nodeOrder.get(b) ?? 0))
1276+
1277+
const result = []
1278+
1279+
while (queue.length > 0) {
1280+
// Take the node with lowest original order (for deterministic output)
1281+
queue.sort((a, b) => (nodeOrder.get(a) ?? 0) - (nodeOrder.get(b) ?? 0))
1282+
const node = queue.shift()
1283+
result.push(node)
1284+
1285+
// Remove edges from this node
1286+
for (const neighbor of adjacency.get(node) || []) {
1287+
inDegree.set(neighbor, inDegree.get(neighbor) - 1)
1288+
if (inDegree.get(neighbor) === 0) {
1289+
queue.push(neighbor)
1290+
}
1291+
}
1292+
}
1293+
1294+
// If we didn't visit all nodes, there's a cycle - return original order
1295+
if (result.length !== dag.nodes.length) {
1296+
console.warn('DAG has cycles, falling back to original order')
1297+
return dag.nodes
1298+
.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
1299+
.map(n => n.blockId)
1300+
}
1301+
1302+
return result
1303+
}
1304+
12451305
// DAG Builder - builds dependency graph from AST analysis
12461306
function buildDAGFromBlocks(blocks) {
12471307
const edges = []
@@ -1356,41 +1416,73 @@ <h3>Run in Browser with WebAssembly</h3>
13561416
})
13571417
}
13581418

1419+
// Browser-mode auto-fix: Replace file reads with in-memory DataFrames
1420+
// This enables cross-notebook data flow without a file system
1421+
function applyBrowserModeTransforms(code, logReplacements = false) {
1422+
return code.replace(
1423+
/pd\.read_csv\s*\(\s*["']([^"']+)\.csv["']\s*\)/g,
1424+
(match, filename) => {
1425+
// Convert filename to likely variable name (e.g., "houses" -> "df_csv")
1426+
const varName = `df_csv` // Common pattern in Deepnote notebooks
1427+
if (logReplacements) {
1428+
console.log(`Browser mode: Replacing pd.read_csv("${filename}.csv") with ${varName}.copy()`)
1429+
}
1430+
return `${varName}.copy() # Browser mode: using in-memory DataFrame instead of file`
1431+
}
1432+
)
1433+
}
1434+
13591435
// Analyze notebook blocks and build DAG
13601436
async function buildNotebookDAG() {
13611437
if (!pyodideReady) return null
13621438

1363-
const notebook = notebookData.project.notebooks[currentNotebookIndex]
1364-
const blocksForAnalyzer = notebook.blocks
1365-
.filter(b => b.type === 'code' || b.type === 'sql' || b.type?.startsWith('input-'))
1366-
.map((block, index) => ({
1367-
type: block.type,
1368-
blockId: block.id,
1369-
code: editedCode.get(block.id) || block.content || '',
1370-
metadata: block.metadata,
1371-
}))
1439+
// Analyze ALL notebooks together for cross-notebook dependencies
1440+
const allBlocksForAnalyzer = []
1441+
let globalOrder = 0
1442+
1443+
notebookData.project.notebooks.forEach((notebook, notebookIndex) => {
1444+
notebook.blocks
1445+
.filter(b => b.type === 'code' || b.type === 'sql' || b.type?.startsWith('input-'))
1446+
.forEach((block) => {
1447+
let code = editedCode.get(block.id) || block.content || ''
1448+
// Apply browser-mode auto-fix for DAG analysis too
1449+
code = applyBrowserModeTransforms(code)
1450+
allBlocksForAnalyzer.push({
1451+
type: block.type,
1452+
blockId: block.id,
1453+
code: code,
1454+
metadata: block.metadata,
1455+
notebookIndex: notebookIndex,
1456+
globalOrder: globalOrder++,
1457+
})
1458+
})
1459+
})
13721460

1373-
if (blocksForAnalyzer.length === 0) return null
1461+
if (allBlocksForAnalyzer.length === 0) return null
13741462

13751463
try {
13761464
// Pass data via Pyodide globals to avoid escaping issues
1377-
pyodide.globals.set('_deepnote_blocks_input', JSON.stringify(blocksForAnalyzer))
1465+
pyodide.globals.set('_deepnote_blocks_input', JSON.stringify(allBlocksForAnalyzer))
13781466
const resultJson = await pyodide.runPythonAsync(`_deepnote_analyze_blocks(_deepnote_blocks_input)`)
13791467
const result = JSON.parse(resultJson)
1380-
1468+
13811469
if (result.errorMessage) {
13821470
console.error('AST analysis error:', result.errorMessage)
13831471
return null
13841472
}
13851473

1386-
// Add order based on original notebook block order
1387-
const blocksWithOrder = result.map(item => ({
1388-
...item,
1389-
order: notebook.blocks.findIndex(b => b.id === item.blockId)
1390-
}))
1474+
// Add order based on global notebook order (preserving cross-notebook dependencies)
1475+
const blocksWithOrder = result.map(item => {
1476+
const inputBlock = allBlocksForAnalyzer.find(b => b.blockId === item.blockId)
1477+
return {
1478+
...item,
1479+
order: inputBlock?.globalOrder ?? 0,
1480+
notebookIndex: inputBlock?.notebookIndex ?? 0,
1481+
}
1482+
})
13911483

13921484
currentDAG = buildDAGFromBlocks(blocksWithOrder)
1393-
console.log('Built DAG:', currentDAG)
1485+
console.log('Built global DAG (all notebooks):', currentDAG)
13941486
return currentDAG
13951487
} catch (error) {
13961488
console.error('Failed to build DAG:', error)
@@ -1443,6 +1535,10 @@ <h3>Run in Browser with WebAssembly</h3>
14431535
await pyodide.loadPackage('micropip')
14441536
const micropip = pyodide.pyimport('micropip')
14451537
await micropip.install('seaborn')
1538+
progressFill.style.width = '90%'
1539+
1540+
messageEl.textContent = 'Installing DuckDB for SQL support...'
1541+
await micropip.install('duckdb')
14461542
progressFill.style.width = '95%'
14471543

14481544
// Setup matplotlib for inline display
@@ -1535,12 +1631,28 @@ <h3>Run in Browser with WebAssembly</h3>
15351631
try {
15361632
// Use edited code if available, otherwise original
15371633
let code = editedCode.get(blockId) || block.content || ''
1538-
1539-
// Skip SQL blocks - they can't run in Pyodide without a database
1634+
1635+
// Apply browser-mode transformations
1636+
code = applyBrowserModeTransforms(code, true) // true = log replacements
1637+
1638+
// Run SQL blocks using DuckDB
15401639
if (block.type === 'sql') {
1541-
outputContainer.innerHTML = '<div style="color: var(--text-muted); font-size: 0.8125rem;">ℹ️ SQL blocks use pre-computed results (no database in browser)</div>'
1542-
blockEl.classList.remove('cell-running')
1543-
return
1640+
const sqlCode = code.trim()
1641+
// Get variable name from block metadata or use default
1642+
const resultVar = block.metadata?.deepnote_variable_name || 'sql_result'
1643+
1644+
// Escape for triple-quoted string (only need to escape triple quotes)
1645+
const escapedSql = sqlCode.replace(/\\/g, '\\\\').replace(/"""/g, '\\"\\"\\"')
1646+
1647+
// Wrap SQL in DuckDB execution
1648+
const wrappedCode = `
1649+
import duckdb
1650+
1651+
# Execute SQL query using DuckDB (can query pandas DataFrames directly)
1652+
${resultVar} = duckdb.query("""${escapedSql}""").df()
1653+
${resultVar}
1654+
`
1655+
code = wrappedCode
15441656
}
15451657

15461658
// Skip bash/shell commands
@@ -1676,9 +1788,11 @@ <h3>Run in Browser with WebAssembly</h3>
16761788

16771789
if (currentDAG) {
16781790
const edgeCount = currentDAG.edges.length
1791+
const notebookCount = new Set(currentDAG.nodes.map(n => n.notebookIndex)).size
1792+
const notebookText = notebookCount > 1 ? ` across ${notebookCount} notebooks` : ''
16791793
indicator.innerHTML = `
16801794
<span style="color: var(--success);">●</span>
1681-
<span>${edgeCount} dependencies tracked</span>
1795+
<span>${edgeCount} dependencies${notebookText}</span>
16821796
`
16831797
} else {
16841798
indicator.innerHTML = `
@@ -1730,30 +1844,48 @@ <h3>Run in Browser with WebAssembly</h3>
17301844
runAllBtn.disabled = true
17311845
runAllBtn.innerHTML = '<svg class="loading-spinner" style="width:16px;height:16px;" viewBox="0 0 24 24"><circle cx="12" cy="12" r="10" stroke="currentColor" stroke-width="2" fill="none" stroke-dasharray="32" stroke-dashoffset="12"></circle></svg><span>Running...</span>'
17321846

1733-
const notebook = notebookData.project.notebooks[currentNotebookIndex]
1847+
// Run ALL cells across ALL notebooks in topological order
1848+
let allCodeBlocks = []
17341849

1735-
// Use DAG order if available, otherwise fall back to sorting key
1736-
let codeBlocks
17371850
if (currentDAG && currentDAG.nodes.length > 0) {
1738-
// Sort blocks by DAG order (topological order based on dependencies)
1851+
// Get all code blocks from all notebooks, sorted by DAG order
17391852
const dagOrder = new Map(currentDAG.nodes.map(n => [n.blockId, n.order]))
1740-
codeBlocks = notebook.blocks
1741-
.filter(b => b.type === 'code' || b.type === 'sql')
1742-
.sort((a, b) => (dagOrder.get(a.id) || 999) - (dagOrder.get(b.id) || 999))
1743-
console.log('Running cells in DAG order:', codeBlocks.map(b => b.id))
1853+
1854+
notebookData.project.notebooks.forEach((notebook) => {
1855+
notebook.blocks
1856+
.filter(b => b.type === 'code' || b.type === 'sql')
1857+
.forEach(block => {
1858+
allCodeBlocks.push({
1859+
...block,
1860+
dagOrder: dagOrder.get(block.id) ?? 999
1861+
})
1862+
})
1863+
})
1864+
1865+
// Perform topological sort based on DAG edges
1866+
const sortedBlockIds = topologicalSort(currentDAG)
1867+
const idToOrder = new Map(sortedBlockIds.map((id, idx) => [id, idx]))
1868+
1869+
// Sort by topological order
1870+
allCodeBlocks.sort((a, b) => (idToOrder.get(a.id) ?? 999) - (idToOrder.get(b.id) ?? 999))
1871+
console.log('Running ALL cells across all notebooks in topological order:', allCodeBlocks.map(b => b.id))
17441872
} else {
1745-
// Fallback to notebook order
1746-
codeBlocks = [...notebook.blocks]
1747-
.sort((a, b) => a.sortingKey.localeCompare(b.sortingKey))
1748-
.filter(b => b.type === 'code' || b.type === 'sql')
1873+
// Fallback: run all notebooks in order
1874+
notebookData.project.notebooks.forEach((notebook) => {
1875+
const blocks = [...notebook.blocks]
1876+
.sort((a, b) => a.sortingKey.localeCompare(b.sortingKey))
1877+
.filter(b => b.type === 'code' || b.type === 'sql')
1878+
allCodeBlocks.push(...blocks)
1879+
})
1880+
console.log('Running ALL cells across all notebooks in notebook order')
17491881
}
17501882

1751-
for (const block of codeBlocks) {
1883+
for (const block of allCodeBlocks) {
17521884
await runCell(block.id)
17531885
}
17541886

17551887
notebookInitialized = true
1756-
console.log('Notebook initialized - all cells have been run')
1888+
console.log('All notebooks initialized - all cells have been run')
17571889

17581890
runAllBtn.disabled = false
17591891
runAllBtn.innerHTML = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polygon points="5 3 19 12 5 21 5 3"></polygon></svg><span>Run All</span>'

0 commit comments

Comments
 (0)