Deepnote

diff --git a/.playwright-mcp/deepnote-code-visible.png b/.playwright-mcp/deepnote-code-visible.png new file mode 100644 index 0000000000..eccf1361c9 Binary files /dev/null and b/.playwright-mcp/deepnote-code-visible.png differ diff --git a/.playwright-mcp/deepnote-data-app.png b/.playwright-mcp/deepnote-data-app.png new file mode 100644 index 0000000000..6e815bc6d5 Binary files /dev/null and b/.playwright-mcp/deepnote-data-app.png differ diff --git a/.playwright-mcp/deepnote-model-training-final.png b/.playwright-mcp/deepnote-model-training-final.png new file mode 100644 index 0000000000..bebaa9b668 Binary files /dev/null and b/.playwright-mcp/deepnote-model-training-final.png differ diff --git a/.playwright-mcp/deepnote-model-training.png b/.playwright-mcp/deepnote-model-training.png new file mode 100644 index 0000000000..98bda41ed6 Binary files /dev/null and b/.playwright-mcp/deepnote-model-training.png differ diff --git a/.playwright-mcp/deepnote-redesigned.png b/.playwright-mcp/deepnote-redesigned.png new file mode 100644 index 0000000000..21a9a1071d Binary files /dev/null and b/.playwright-mcp/deepnote-redesigned.png differ diff --git a/.playwright-mcp/deepnote-run-hidden.png b/.playwright-mcp/deepnote-run-hidden.png new file mode 100644 index 0000000000..2bd808f213 Binary files /dev/null and b/.playwright-mcp/deepnote-run-hidden.png differ diff --git a/.playwright-mcp/deepnote-with-chart.png b/.playwright-mcp/deepnote-with-chart.png new file mode 100644 index 0000000000..441cc20bf8 Binary files /dev/null and b/.playwright-mcp/deepnote-with-chart.png differ diff --git a/.playwright-mcp/editable-code.png b/.playwright-mcp/editable-code.png new file mode 100644 index 0000000000..707fc0967a Binary files /dev/null and b/.playwright-mcp/editable-code.png differ diff --git a/.playwright-mcp/pyodide-matplotlib.png b/.playwright-mcp/pyodide-matplotlib.png new file mode 100644 index 0000000000..89691ef958 Binary files /dev/null and b/.playwright-mcp/pyodide-matplotlib.png differ diff --git a/.playwright-mcp/pyodide-working.png b/.playwright-mcp/pyodide-working.png new file mode 100644 index 0000000000..cb79319c9c Binary files /dev/null and b/.playwright-mcp/pyodide-working.png differ diff --git a/.playwright-mcp/reactivity-working.png b/.playwright-mcp/reactivity-working.png new file mode 100644 index 0000000000..a2cfd10d48 Binary files /dev/null and b/.playwright-mcp/reactivity-working.png differ diff --git a/.playwright-mcp/sql-graceful-skip.png b/.playwright-mcp/sql-graceful-skip.png new file mode 100644 index 0000000000..46d0e7a945 Binary files /dev/null and b/.playwright-mcp/sql-graceful-skip.png differ diff --git a/data-app/index.html b/data-app/index.html new file mode 100644 index 0000000000..17c7a7553a --- /dev/null +++ b/data-app/index.html @@ -0,0 +1,2457 @@ + + + + + + Deepnote + + + + + + + + +

+ + + + +

+ + +

+ + + + + +

Loading notebook...

+ + + + diff --git a/data-app/package-lock.json b/data-app/package-lock.json new file mode 100644 index 0000000000..efed2c4c56 --- /dev/null +++ b/data-app/package-lock.json @@ -0,0 +1,59 @@ +{ + "name": "deepnote-data-app", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "deepnote-data-app", + "version": "1.0.0", + "dependencies": { + "chokidar": "^4.0.3", + "yaml": "^2.6.1" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/yaml": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", + "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/data-app/package.json b/data-app/package.json new file mode 100644 index 0000000000..68823e1317 --- /dev/null +++ b/data-app/package.json @@ -0,0 +1,12 @@ +{ + "name": "deepnote-data-app", + "version": "1.0.0", + "type": "module", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "chokidar": "^4.0.3", + "yaml": "^2.6.1" + } +} diff --git a/data-app/server.js b/data-app/server.js new file mode 100644 index 0000000000..273deafb8a --- /dev/null +++ b/data-app/server.js @@ -0,0 +1,70 @@ +import { watch } from 'node:fs' +import { readFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parse as parseYaml } from 'yaml' + +const __dirname = dirname(fileURLToPath(import.meta.url)) + +// Path to the .deepnote file to display +const DEEPNOTE_FILE = join(__dirname, '../examples/demos/housing_price_prediction.deepnote') + +// Store SSE clients for live updates +const clients = new Set() + +// Watch the file for changes +watch(DEEPNOTE_FILE, eventType => { + if (eventType === 'change') { + console.log('📄 File changed, notifying clients...') + for (const client of clients) { + client.write('data: reload\n\n') + } + } +}) + +async function loadDeepnoteFile() { + const content = await readFile(DEEPNOTE_FILE, 'utf-8') + return parseYaml(content) +} + +const server = createServer(async (req, res) => { + const url = new URL(req.url, `http://${req.headers.host}`) + + // CORS headers + res.setHeader('Access-Control-Allow-Origin', '*') + + if (url.pathname === '/') { + const html = await readFile(join(__dirname, 'index.html'), 'utf-8') + res.writeHead(200, { 'Content-Type': 'text/html' }) + res.end(html) + } else if (url.pathname === '/api/notebook') { + try { + const data = await loadDeepnoteFile() + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify(data)) + } catch (err) { + res.writeHead(500, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: err.message })) + } + } else if (url.pathname === '/events') { + // Server-Sent Events for live reload + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }) + clients.add(res) + req.on('close', () => clients.delete(res)) + } else { + res.writeHead(404) + res.end('Not found') + } +}) + +const PORT = 3456 +server.listen(PORT, () => { + console.log(`\n🚀 Deepnote Data App running at http://localhost:${PORT}`) + console.log(`📊 Displaying: ${DEEPNOTE_FILE}`) + console.log(`👀 Watching for file changes...\n`) +}) diff --git a/packages/reactivity/README.md b/packages/reactivity/README.md new file mode 100644 index 0000000000..2b3918385b --- /dev/null +++ b/packages/reactivity/README.md @@ -0,0 +1,56 @@ +# @deepnote/reactivity + +Reactivity and dependency graph for Deepnote notebooks. This package provides: + +1. **AST Analyzer** - Parses Python code to extract variable definitions and usages +2. **DAG Builder** - Builds a directed acyclic graph of cell dependencies +3. **DAG Analyzer** - Finds downstream cells that need to re-run when a cell changes + +## Usage + +### In Node.js (with child process) + +```typescript +import { getBlocksContentDeps } from "@deepnote/reactivity"; + +const blocks = [ + { cellId: "a", cell_type: "code", source: "x = 1" }, + { cellId: "b", cell_type: "code", source: "y = x + 1" }, +]; + +const deps = await getBlocksContentDeps(blocks); +``` + +### In Browser (with Pyodide) + +```typescript +import { + createBrowserAstAnalyzer, + buildDAGFromBlocks, + getDownstreamBlocksForBlocksIds, +} from "@deepnote/reactivity/browser"; + +// Initialize with Pyodide instance +const analyzer = createBrowserAstAnalyzer(pyodide); + +// Analyze blocks +const deps = await analyzer.analyzeBlocks(blocks); + +// Build DAG +const dag = buildDAGFromBlocks(deps); + +// Get downstream blocks when cell 'a' changes +const downstream = getDownstreamBlocksForBlocksIds(dag, ["a"]); +``` + +## Features + +- Full Python AST parsing for variable extraction +- SQL block support with Jinja variable detection +- Input block support (text, select, slider, etc.) +- Efficient DAG traversal for reactive updates +- Browser-compatible via Pyodide + +## License + +MIT diff --git a/packages/reactivity/package.json b/packages/reactivity/package.json new file mode 100644 index 0000000000..1baffa1bfb --- /dev/null +++ b/packages/reactivity/package.json @@ -0,0 +1,36 @@ +{ + "name": "@deepnote/reactivity", + "version": "0.1.0", + "description": "Reactivity and dependency graph for Deepnote notebooks", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs", + "types": "./dist/index.d.ts" + }, + "./browser": { + "import": "./dist/browser.js", + "require": "./dist/browser.cjs", + "types": "./dist/browser.d.ts" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsdown src/index.ts src/browser.ts --format cjs,esm --dts --clean", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "zod": "3.25.76" + }, + "engines": { + "node": ">=22.14.0", + "pnpm": ">=10.17.1" + }, + "license": "MIT" +} diff --git a/packages/reactivity/src/ast-analyzer.py b/packages/reactivity/src/ast-analyzer.py new file mode 100644 index 0000000000..7fbfc36a94 --- /dev/null +++ b/packages/reactivity/src/ast-analyzer.py @@ -0,0 +1,383 @@ +""" +AST Analyzer for Python code blocks. +Extracts variable definitions and usages for dependency graph construction. +This version is designed to run in Pyodide (browser) without file I/O. +""" + +import ast +import json +import re + + +class VariableVisitor(ast.NodeVisitor): + def __init__(self): + self.global_vars = set() # Variables defined globally + self.used_global_vars = set() # Variables used and defined globally + self.imported_modules = set() # Imported modules + self.scope_stack = [] # Stack to track scopes + self.function_globals = set() # Global variables declared in current function + + def current_scope_is_global(self): + # If the scope stack is empty, we are at the global level + return not self.scope_stack + + def visit_Global(self, node): + for name in node.names: + self.function_globals.add(name) + self.generic_visit(node) + + def visit_Assign(self, node): + for target in node.targets: + if isinstance(target, ast.Name): + if self.current_scope_is_global(): + self.global_vars.add(target.id) + self.generic_visit(node) + + def visit_AugAssign(self, node): + if isinstance(node.target, ast.Name): + if self.current_scope_is_global(): + self.global_vars.add(node.target.id) + self.generic_visit(node) + + def visit_AnnAssign(self, node): + target = node.target + if isinstance(target, ast.Name) and self.current_scope_is_global(): + self.global_vars.add(target.id) + self.generic_visit(node) + + def visit_NamedExpr(self, node): + if isinstance(node.target, ast.Name) and self.current_scope_is_global(): + self.global_vars.add(node.target.id) + self.generic_visit(node) + + def visit_ClassDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + self.scope_stack.append(node.name) # Enter class scope + self.generic_visit(node) + self.scope_stack.pop() # Exit class scope + + def visit_FunctionDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + + prev_function_globals = self.function_globals + self.function_globals = set() + + self.scope_stack.append(node.name) # Enter function scope + self.generic_visit(node) + self.scope_stack.pop() # Exit function scope + + self.function_globals = prev_function_globals + + def visit_AsyncFunctionDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + + prev_function_globals = self.function_globals + self.function_globals = set() + + self.scope_stack.append(node.name) # Enter function scope + self.generic_visit(node) + self.scope_stack.pop() # Exit function scope + + self.function_globals = prev_function_globals + + def visit_Name(self, node): + builtins = { + "print", + "len", + "range", + "str", + "int", + "float", + "bool", + "list", + "dict", + "set", + "tuple", + "abs", + "all", + "any", + "bin", + "callable", + "chr", + "dir", + "enumerate", + "eval", + "exec", + "filter", + "format", + "getattr", + "globals", + "hasattr", + "hash", + "help", + "hex", + "id", + "input", + "isinstance", + "issubclass", + "iter", + "locals", + "map", + "max", + "min", + "next", + "oct", + "open", + "ord", + "pow", + "repr", + "reversed", + "round", + "setattr", + "sorted", + "sum", + "type", + "vars", + "zip", + "__import__", + "True", + "False", + "None", + } + + if isinstance(node.ctx, ast.Load): + if node.id in builtins: + self.generic_visit(node) + return + + if self.current_scope_is_global(): + self.used_global_vars.add(node.id) + elif node.id in self.function_globals: + # Variable explicitly declared as global in current function + self.used_global_vars.add(node.id) + elif node.id in self.global_vars: + self.used_global_vars.add(node.id) + elif isinstance(node.ctx, ast.Store): + # Only track variable assignments at global scope + if self.current_scope_is_global(): + self.global_vars.add(node.id) + self.generic_visit(node) + + def visit_Attribute(self, node): + # Attributes are part of global usage if they are prefixed by a global variable + if isinstance(node.value, ast.Name): + if self.current_scope_is_global(): + self.used_global_vars.add(node.value.id) + elif node.value.id in self.function_globals: + # Variable explicitly declared as global in current function + self.used_global_vars.add(node.value.id) + elif node.value.id in self.global_vars: + self.used_global_vars.add(node.value.id) + else: + self.generic_visit(node) + + def visit_Import(self, node): + for alias in node.names: + self.imported_modules.add(alias.asname or alias.name) + + def visit_ImportFrom(self, node): + for alias in node.names: + self.imported_modules.add(alias.asname or alias.name) + + +def get_defined_used_variables(code): + visitor = VariableVisitor() + tree = ast.parse(code) + visitor.visit(tree) + return ( + visitor.global_vars, + visitor.used_global_vars, + visitor.imported_modules, + ) + + +def sanitize_python_variable_name(name): + """ + Python implementation of sanitizePythonVariableName from utils.ts + """ + sanitized = re.sub(r"\s+", "_", name) + sanitized = re.sub(r"[^0-9a-zA-Z_]", "", sanitized) + sanitized = re.sub(r"^[^a-zA-Z_]+", "", sanitized) + + if sanitized == "": + sanitized = "input_1" + + return sanitized + + +def extract_jinja_variables(sql_code): + """ + Extract Jinja variables from SQL code. + Simplified version that doesn't require jinja2 - just uses regex. + """ + # Find {{ variable }} patterns + jinja_vars = set(re.findall(r"\{\{\s*(\w+)", sql_code)) + + # Look for table names after FROM, JOIN, etc. that could be variables + clean_sql = re.sub(r"\{\{.*?\}\}", "", sql_code) + clean_sql = re.sub(r"\{%.*?%\}", "", clean_sql, flags=re.DOTALL) + + table_patterns = [ + r"\bFROM\s+([a-zA-Z_][a-zA-Z0-9_]*)", + r"\bJOIN\s+([a-zA-Z_][a-zA-Z0-9_]*)", + r"\bINTO\s+([a-zA-Z_][a-zA-Z0-9_]*)", + r"\bUPDATE\s+([a-zA-Z_][a-zA-Z0-9_]*)", + ] + + sql_keywords = { + "select", + "where", + "group", + "order", + "having", + "limit", + "offset", + "union", + "intersect", + "except", + } + + for pattern in table_patterns: + matches = re.findall(pattern, clean_sql, re.IGNORECASE) + for match in matches: + if match.lower() not in sql_keywords: + jinja_vars.add(match) + + return jinja_vars + + +def comment_out_jupyter_bash_commands(code): + """Comment out lines starting with % or ! to prevent parse errors.""" + lines = code.split("\n") + for i in range(len(lines)): + if lines[i].startswith("%") or lines[i].startswith("!"): + lines[i] = "#" + lines[i] + return "\n".join(lines) + + +def analyze_block(block): + """Analyze a single block and return its variable dependencies.""" + try: + block_type = block.get("type") or "code" + code = block.get("code", "") + metadata = block.get("metadata", {}) or {} + block_id = block["blockId"] + + if block_type == "code": + # Comment out magic commands + code = comment_out_jupyter_bash_commands(code) + block_defined, block_used, block_imported = get_defined_used_variables(code) + + return { + "blockId": block_id, + "definedVariables": sorted(list(block_defined)), + "usedVariables": sorted(list(block_used)), + "importedModules": list(block_imported), + } + + elif block_type == "sql": + jinja_variables = extract_jinja_variables(code) + output_variables = [] + if metadata.get("deepnote_variable_name"): + output_variables = [metadata["deepnote_variable_name"]] + + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": sorted(list(jinja_variables)), + "importedModules": [], + } + + elif block_type == "button": + output_variables = [] + if metadata.get("deepnote_variable_name"): + output_variables = [metadata["deepnote_variable_name"]] + + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": [], + "importedModules": [], + } + + elif block_type == "big-number": + used_variables = [] + if metadata.get("deepnote_big_number_value"): + used_variables.append(metadata["deepnote_big_number_value"]) + if metadata.get("deepnote_big_number_comparison_value"): + used_variables.append(metadata["deepnote_big_number_comparison_value"]) + + return { + "blockId": block_id, + "definedVariables": [], + "usedVariables": sorted(list(set(used_variables))), + "importedModules": [], + } + + elif block_type.startswith("input-"): + output_variables = [] + if metadata.get("deepnote_variable_name"): + variable_name = metadata["deepnote_variable_name"] + if variable_name: + sanitized_name = sanitize_python_variable_name(variable_name) + output_variables = [sanitized_name] + + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": [], + "importedModules": [], + } + + else: + # Unknown block type - return empty + return { + "blockId": block_id, + "definedVariables": [], + "usedVariables": [], + "importedModules": [], + } + + except Exception as e: + return { + "blockId": block.get("blockId", "unknown"), + "definedVariables": [], + "usedVariables": [], + "importedModules": [], + "error": { + "type": e.__class__.__name__, + "message": str(e), + }, + } + + +def analyze_blocks(blocks_json): + """ + Main entry point for browser usage. + Takes JSON string of blocks, returns JSON string of analysis. + """ + try: + blocks = ( + json.loads(blocks_json) if isinstance(blocks_json, str) else blocks_json + ) + results = [analyze_block(block) for block in blocks] + return json.dumps(results) + except Exception as e: + return json.dumps({"errorMessage": f"{e.__class__.__name__}: {str(e)}"}) + + +# For direct testing +if __name__ == "__main__": + test_blocks = [ + {"blockId": "a", "type": "code", "code": "x = 1\ny = 2"}, + {"blockId": "b", "type": "code", "code": "z = x + y"}, + { + "blockId": "c", + "type": "code", + "code": "import pandas as pd\ndf = pd.DataFrame()", + }, + ] + print(analyze_blocks(json.dumps(test_blocks))) diff --git a/packages/reactivity/src/browser.ts b/packages/reactivity/src/browser.ts new file mode 100644 index 0000000000..50e14323ca --- /dev/null +++ b/packages/reactivity/src/browser.ts @@ -0,0 +1,374 @@ +/** + * Browser-compatible reactivity module. + * Uses Pyodide to run Python AST analysis in the browser. + */ + +import { getDownstreamBlocksForBlocksIds } from './dag-analyzer' +import { buildDAGFromBlocks } from './dag-builder' +import type { AnalyzerBlock, AstAnalyzerItem, BlockContentDepsDAG, BlockContentDepsWithOrder } from './types' +import { AstAnalyzerResponseSchema } from './types' + +export { getDownstreamBlocksForBlocksIds } from './dag-analyzer' +// Re-export DAG utilities +export { buildDAGFromBlocks } from './dag-builder' +export type { + AnalyzerBlock, + AstAnalyzerItem, + BlockContentDeps, + BlockContentDepsDAG, + BlockContentDepsWithOrder, + DAGEdge, + DAGNode, +} from './types' + +// The Python AST analyzer code (will be loaded at runtime) +const AST_ANALYZER_CODE = ` +import ast +import json +import re + + +class VariableVisitor(ast.NodeVisitor): + def __init__(self): + self.global_vars = set() + self.used_global_vars = set() + self.imported_modules = set() + self.scope_stack = [] + self.function_globals = set() + + def current_scope_is_global(self): + return not self.scope_stack + + def visit_Global(self, node): + for name in node.names: + self.function_globals.add(name) + self.generic_visit(node) + + def visit_Assign(self, node): + for target in node.targets: + if isinstance(target, ast.Name): + if self.current_scope_is_global(): + self.global_vars.add(target.id) + self.generic_visit(node) + + def visit_AugAssign(self, node): + if isinstance(node.target, ast.Name): + if self.current_scope_is_global(): + self.global_vars.add(node.target.id) + self.generic_visit(node) + + def visit_AnnAssign(self, node): + target = node.target + if isinstance(target, ast.Name) and self.current_scope_is_global(): + self.global_vars.add(target.id) + self.generic_visit(node) + + def visit_NamedExpr(self, node): + if isinstance(node.target, ast.Name) and self.current_scope_is_global(): + self.global_vars.add(node.target.id) + self.generic_visit(node) + + def visit_ClassDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + self.scope_stack.append(node.name) + self.generic_visit(node) + self.scope_stack.pop() + + def visit_FunctionDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + prev_function_globals = self.function_globals + self.function_globals = set() + self.scope_stack.append(node.name) + self.generic_visit(node) + self.scope_stack.pop() + self.function_globals = prev_function_globals + + def visit_AsyncFunctionDef(self, node): + if self.current_scope_is_global(): + self.global_vars.add(node.name) + prev_function_globals = self.function_globals + self.function_globals = set() + self.scope_stack.append(node.name) + self.generic_visit(node) + self.scope_stack.pop() + self.function_globals = prev_function_globals + + def visit_Name(self, node): + builtins = { + 'print', 'len', 'range', 'str', 'int', 'float', 'bool', 'list', 'dict', 'set', 'tuple', + 'abs', 'all', 'any', 'bin', 'callable', 'chr', 'dir', 'enumerate', 'eval', 'exec', + 'filter', 'format', 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', + 'input', 'isinstance', 'issubclass', 'iter', 'locals', 'map', 'max', 'min', 'next', + 'oct', 'open', 'ord', 'pow', 'repr', 'reversed', 'round', 'setattr', 'sorted', 'sum', + 'type', 'vars', 'zip', '__import__', 'True', 'False', 'None' + } + if isinstance(node.ctx, ast.Load): + if node.id in builtins: + self.generic_visit(node) + return + if self.current_scope_is_global(): + self.used_global_vars.add(node.id) + elif node.id in self.function_globals: + self.used_global_vars.add(node.id) + elif node.id in self.global_vars: + self.used_global_vars.add(node.id) + elif isinstance(node.ctx, ast.Store): + if self.current_scope_is_global(): + self.global_vars.add(node.id) + self.generic_visit(node) + + def visit_Attribute(self, node): + if isinstance(node.value, ast.Name): + if self.current_scope_is_global(): + self.used_global_vars.add(node.value.id) + elif node.value.id in self.function_globals: + self.used_global_vars.add(node.value.id) + elif node.value.id in self.global_vars: + self.used_global_vars.add(node.value.id) + else: + self.generic_visit(node) + + def visit_Import(self, node): + for alias in node.names: + self.imported_modules.add(alias.asname or alias.name) + + def visit_ImportFrom(self, node): + for alias in node.names: + self.imported_modules.add(alias.asname or alias.name) + + +def _get_defined_used_variables(code): + visitor = VariableVisitor() + tree = ast.parse(code) + visitor.visit(tree) + return (visitor.global_vars, visitor.used_global_vars, visitor.imported_modules) + + +def _sanitize_python_variable_name(name): + sanitized = re.sub(r'\\s+', '_', name) + sanitized = re.sub(r'[^0-9a-zA-Z_]', '', sanitized) + sanitized = re.sub(r'^[^a-zA-Z_]+', '', sanitized) + if sanitized == '': + sanitized = 'input_1' + return sanitized + + +def _extract_jinja_variables(sql_code): + jinja_vars = set(re.findall(r'\\{\\{\\s*(\\w+)', sql_code)) + clean_sql = re.sub(r'\\{\\{.*?\\}\\}', '', sql_code) + clean_sql = re.sub(r'\\{%.*?%\\}', '', clean_sql, flags=re.DOTALL) + table_patterns = [ + r'\\bFROM\\s+([a-zA-Z_][a-zA-Z0-9_]*)', + r'\\bJOIN\\s+([a-zA-Z_][a-zA-Z0-9_]*)', + r'\\bINTO\\s+([a-zA-Z_][a-zA-Z0-9_]*)', + r'\\bUPDATE\\s+([a-zA-Z_][a-zA-Z0-9_]*)' + ] + sql_keywords = {'select', 'where', 'group', 'order', 'having', 'limit', 'offset', 'union', 'intersect', 'except'} + for pattern in table_patterns: + matches = re.findall(pattern, clean_sql, re.IGNORECASE) + for match in matches: + if match.lower() not in sql_keywords: + jinja_vars.add(match) + return jinja_vars + + +def _comment_out_jupyter_bash_commands(code): + lines = code.split("\\n") + for i in range(len(lines)): + if lines[i].startswith("%") or lines[i].startswith("!"): + lines[i] = "#" + lines[i] + return "\\n".join(lines) + + +def _analyze_block(block): + try: + block_type = block.get("type") or "code" + code = block.get("code", "") + metadata = block.get("metadata", {}) or {} + block_id = block["blockId"] + + if block_type == "code": + code = _comment_out_jupyter_bash_commands(code) + block_defined, block_used, block_imported = _get_defined_used_variables(code) + return { + "blockId": block_id, + "definedVariables": sorted(list(block_defined)), + "usedVariables": sorted(list(block_used)), + "importedModules": list(block_imported), + } + elif block_type == "sql": + jinja_variables = _extract_jinja_variables(code) + output_variables = [] + if metadata.get("deepnote_variable_name"): + output_variables = [metadata["deepnote_variable_name"]] + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": sorted(list(jinja_variables)), + "importedModules": [], + } + elif block_type == "button": + output_variables = [] + if metadata.get("deepnote_variable_name"): + output_variables = [metadata["deepnote_variable_name"]] + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": [], + "importedModules": [], + } + elif block_type == "big-number": + used_variables = [] + if metadata.get("deepnote_big_number_value"): + used_variables.append(metadata["deepnote_big_number_value"]) + if metadata.get("deepnote_big_number_comparison_value"): + used_variables.append(metadata["deepnote_big_number_comparison_value"]) + return { + "blockId": block_id, + "definedVariables": [], + "usedVariables": sorted(list(set(used_variables))), + "importedModules": [], + } + elif block_type.startswith("input-"): + output_variables = [] + if metadata.get("deepnote_variable_name"): + variable_name = metadata["deepnote_variable_name"] + if variable_name: + sanitized_name = _sanitize_python_variable_name(variable_name) + output_variables = [sanitized_name] + return { + "blockId": block_id, + "definedVariables": output_variables, + "usedVariables": [], + "importedModules": [], + } + else: + return { + "blockId": block_id, + "definedVariables": [], + "usedVariables": [], + "importedModules": [], + } + except Exception as e: + return { + "blockId": block.get("blockId", "unknown"), + "definedVariables": [], + "usedVariables": [], + "importedModules": [], + "error": {"type": e.__class__.__name__, "message": str(e)}, + } + + +def _deepnote_analyze_blocks(blocks_json): + try: + blocks = json.loads(blocks_json) if isinstance(blocks_json, str) else blocks_json + results = [_analyze_block(block) for block in blocks] + return json.dumps(results) + except Exception as e: + return json.dumps({"errorMessage": f"{e.__class__.__name__}: {str(e)}"}) +` + +// Pyodide interface (minimal typing) +interface PyodideInterface { + runPythonAsync: (code: string) => Promise + globals: { + get: (name: string) => unknown + } +} + +/** + * Creates a browser-compatible AST analyzer that runs in Pyodide. + */ +export function createBrowserAstAnalyzer(pyodide: PyodideInterface) { + let initialized = false + + async function ensureInitialized(): Promise { + if (initialized) return + + // Load the AST analyzer code into Pyodide + await pyodide.runPythonAsync(AST_ANALYZER_CODE) + initialized = true + } + + /** + * Analyze blocks and return their variable dependencies. + */ + async function analyzeBlocks(blocks: AnalyzerBlock[]): Promise { + await ensureInitialized() + + // Convert blocks to the format expected by the Python analyzer + const blocksForAnalyzer = blocks + .filter( + block => + block.cell_type === 'code' || + block.cell_type === 'sql' || + block.cell_type === 'button' || + block.cell_type === 'big-number' || + block.cell_type?.startsWith('input-') + ) + .map(block => ({ + type: block.cell_type, + blockId: block.cellId, + code: block.source, + metadata: block.metadata, + })) + + if (blocksForAnalyzer.length === 0) { + return [] + } + + const inputJson = JSON.stringify(blocksForAnalyzer) + + // Call the Python analyzer + const resultJson = (await pyodide.runPythonAsync( + `_deepnote_analyze_blocks('''${inputJson.replace(/'/g, "\\'")}''')` + )) as string + + const parsed = AstAnalyzerResponseSchema.safeParse(JSON.parse(resultJson)) + if (!parsed.success) { + throw new Error('Invalid AST analyzer response format') + } + + const parsedData = parsed.data + + if (Array.isArray(parsedData)) { + return parsedData.map((item: AstAnalyzerItem) => ({ + ...item, + order: blocks.findIndex(b => b.cellId === item.blockId), + })) + } + + if ('errorMessage' in parsedData) { + throw new Error(parsedData.errorMessage) + } + + throw new Error('Unexpected AST analyzer response') + } + + /** + * Build a complete DAG from blocks. + */ + async function buildDAG(blocks: AnalyzerBlock[]): Promise { + const deps = await analyzeBlocks(blocks) + return buildDAGFromBlocks(deps) + } + + /** + * Get all downstream blocks that depend on the given block IDs. + */ + async function getDownstreamBlocks(blocks: AnalyzerBlock[], changedBlockIds: string[]): Promise { + const dag = await buildDAG(blocks) + return getDownstreamBlocksForBlocksIds(dag, changedBlockIds) + } + + return { + analyzeBlocks, + buildDAG, + getDownstreamBlocks, + ensureInitialized, + } +} + +export type BrowserAstAnalyzer = ReturnType diff --git a/packages/reactivity/src/dag-analyzer.ts b/packages/reactivity/src/dag-analyzer.ts new file mode 100644 index 0000000000..7b36926e1e --- /dev/null +++ b/packages/reactivity/src/dag-analyzer.ts @@ -0,0 +1,36 @@ +import type { BlockContentDepsDAG } from './types' + +/** + * Accepts blockIds of changed inputs and returns ids of blocks that use these inputs (dependencies). + * The method finds dependencies in downstream mode - we are filtering out nodes from the DAG that are defined before the changed blocks. + */ +export function getDownstreamBlocksForBlocksIds(dag: BlockContentDepsDAG, blocksIds: string[]): string[] { + const changedBlocks = dag.nodes.filter(node => blocksIds.includes(node.blockId)) + const minOrder = Math.min(...changedBlocks.map(node => node.order)) + const filteredDag = { + ...dag, + nodes: dag.nodes.filter(node => node.order >= minOrder), + } + + return getDownstreamBlocks(filteredDag, blocksIds) +} + +function getDownstreamBlocks(dag: BlockContentDepsDAG, blocksIds: string[], visited = new Set()): string[] { + const outputVariables = dag.nodes + .filter(node => blocksIds.includes(node.blockId)) + .flatMap(node => node.outputVariables) + + const blocksThatAreUsingVariables = dag.nodes + .filter(node => node.inputVariables.find(inputVariable => outputVariables.includes(inputVariable))) + .map(node => node.blockId) + + if (blocksThatAreUsingVariables.length === 0) { + return [] + } + + // This prevents the function from making recursive calls for blocks that have already been visited. + const nextBlocks = blocksThatAreUsingVariables.filter(blockId => !visited.has(blockId)) + nextBlocks.forEach(blockId => visited.add(blockId)) + + return [...nextBlocks, ...getDownstreamBlocks(dag, nextBlocks, visited)] +} diff --git a/packages/reactivity/src/dag-builder.ts b/packages/reactivity/src/dag-builder.ts new file mode 100644 index 0000000000..adc9f78f10 --- /dev/null +++ b/packages/reactivity/src/dag-builder.ts @@ -0,0 +1,168 @@ +import type { BlockContentDepsDAG, BlockContentDepsWithOrder, DAGEdge, DAGNode } from './types' + +/** + * Goes through all blocks and creates DAG. + * - usedVariables can contain used modules. We have to traverse other blocks in the list to find whether a variable is a module usage. + * - outputVariables don't contain variables defined by import statements like `pd` from `import pandas as pd` - those are in importedModules. + * + * The modules dependencies are intentionally excluded to improve reactivity performance. The reactivity in app works only after + * initial (top-to-bottom) run and after that all the imports should be present so there is no need re-run the blocks that use them or depend on them. + * We are still creating `modulesEdges` which are visualized as dashed lines in the DAG graph. + */ +export function buildDAGFromBlocks(blocks: BlockContentDepsWithOrder[]): BlockContentDepsDAG { + const edges: DAGEdge[] = [] + const modulesEdges: DAGEdge[] = [] + + const moduleDefinitionsLookup: Record = {} + blocks + .flatMap(block => block.importedModules ?? []) + .forEach(m => { + moduleDefinitionsLookup[m] = true + }) + + const blocksWithUsedImportedModules = blocks.map(block => { + const usedImportedModules = block.usedVariables.filter( + (variable: string) => moduleDefinitionsLookup[variable] === true + ) + const usedVariables = block.usedVariables.filter((variable: string) => moduleDefinitionsLookup[variable] !== true) + + return { + ...block, + usedVariables, + usedImportedModules, + } + }) + + const nodes: Record = {} + blocksWithUsedImportedModules.forEach(block => { + nodes[block.blockId] = { + blockId: block.blockId, + inputVariables: [], + order: block.order, + importedModules: block.importedModules ?? [], + outputVariables: [...block.definedVariables], + usedImportedModules: block.usedImportedModules ?? [], + error: block.error ?? null, + } + }) + + // We are not simply adding all edges based on usedVariables to definedVariables. + // We are taking block order into account - creating edges only with the closest defining block + // or the closest block that uses the variable. When traversing the DAG, we should still + // get the same result as if we were adding all edges. + blocksWithUsedImportedModules.forEach(block => { + // Nodes for all blocks are created above; safe to assert existence when indexing + const currentNode = nodes[block.blockId] + if (!currentNode) { + return + } + + block.usedVariables.forEach((usedVar: string) => { + const [closestDefiningBlock] = blocksWithUsedImportedModules + .filter(b => b.definedVariables.includes(usedVar) && b.order < block.order) + .sort((a, b) => b.order - a.order) + + const [closesBlockThatUsesVariable] = blocksWithUsedImportedModules + .filter(b => b.usedVariables.includes(usedVar) && b.order < block.order) + .sort((a, b) => b.order - a.order) + + // If there is a block that defines the variable and a block that uses the variable, + // we want to create an edge between the closest one. + if (closestDefiningBlock && closesBlockThatUsesVariable) { + if (closestDefiningBlock.order > closesBlockThatUsesVariable.order) { + if (!currentNode.inputVariables.includes(usedVar)) { + currentNode.inputVariables.push(usedVar) + } + const definingNode = nodes[closestDefiningBlock.blockId] + if (!definingNode) { + return + } + if (!definingNode.outputVariables.includes(usedVar)) { + definingNode.outputVariables.push(usedVar) + } + + edges.push({ + from: closestDefiningBlock.blockId, + to: block.blockId, + inputVariables: [usedVar], + }) + } else { + if (!currentNode.inputVariables.includes(usedVar)) { + currentNode.inputVariables.push(usedVar) + } + const usingNode = nodes[closesBlockThatUsesVariable.blockId] + if (!usingNode) { + return + } + if (!usingNode.outputVariables.includes(usedVar)) { + usingNode.outputVariables.push(usedVar) + } + + edges.push({ + from: closesBlockThatUsesVariable.blockId, + to: block.blockId, + inputVariables: [usedVar], + }) + } + } else { + if (closestDefiningBlock) { + if (!currentNode.inputVariables.includes(usedVar)) { + currentNode.inputVariables.push(usedVar) + } + const definingNode = nodes[closestDefiningBlock.blockId] + if (!definingNode) { + return + } + if (!definingNode.outputVariables.includes(usedVar)) { + definingNode.outputVariables.push(usedVar) + } + + edges.push({ + from: closestDefiningBlock.blockId, + to: block.blockId, + inputVariables: [usedVar], + }) + } + + if (closesBlockThatUsesVariable) { + if (!currentNode.inputVariables.includes(usedVar)) { + currentNode.inputVariables.push(usedVar) + } + const usingNode = nodes[closesBlockThatUsesVariable.blockId] + if (!usingNode) { + return + } + if (!usingNode.outputVariables.includes(usedVar)) { + usingNode.outputVariables.push(usedVar) + } + + edges.push({ + from: closesBlockThatUsesVariable.blockId, + to: block.blockId, + inputVariables: [usedVar], + }) + } + } + }) + + block.usedImportedModules?.forEach((usedModule: string) => { + const closestDefiningBlock = blocksWithUsedImportedModules.find(b => + (b.importedModules ?? []).includes(usedModule) + ) + + if (closestDefiningBlock) { + modulesEdges.push({ + from: closestDefiningBlock.blockId, + to: block.blockId, + inputVariables: [usedModule], + }) + } + }) + }) + + return { + nodes: Object.values(nodes), + edges, + modulesEdges, + } +} diff --git a/packages/reactivity/src/index.ts b/packages/reactivity/src/index.ts new file mode 100644 index 0000000000..0331d65db5 --- /dev/null +++ b/packages/reactivity/src/index.ts @@ -0,0 +1,25 @@ +/** + * @deepnote/reactivity + * + * Reactivity and dependency graph for Deepnote notebooks. + */ + +export { getDownstreamBlocksForBlocksIds } from './dag-analyzer' +export { buildDAGFromBlocks } from './dag-builder' +export type { + AnalyzerBlock, + AstAnalyzerItem, + BlockContentDeps, + BlockContentDepsDAG, + BlockContentDepsWithOrder, + BlockError, + DAGEdge, + DAGNode, +} from './types' +export { + AstAnalyzerErrorSchema, + AstAnalyzerItemSchema, + AstAnalyzerResponseSchema, + AstAnalyzerSuccessSchema, + BlockContentDepsSchema, +} from './types' diff --git a/packages/reactivity/src/types.ts b/packages/reactivity/src/types.ts new file mode 100644 index 0000000000..47297f4e0d --- /dev/null +++ b/packages/reactivity/src/types.ts @@ -0,0 +1,73 @@ +import { z } from 'zod' + +const BlockErrorSchema = z.object({ + type: z.string(), + message: z.string(), +}) + +export type BlockError = z.infer + +export const BlockContentDepsSchema = z.object({ + blockId: z.string(), + definedVariables: z.array(z.string()), + usedVariables: z.array(z.string()), + importedModules: z.array(z.string()).optional(), + usedImportedModules: z.array(z.string()).optional(), + error: BlockErrorSchema.optional(), +}) + +export type BlockContentDeps = z.infer + +export interface BlockContentDepsWithOrder extends BlockContentDeps { + order: number +} + +export interface DAGNode { + blockId: string + inputVariables: string[] + importedModules: string[] + order: number + outputVariables: string[] + usedImportedModules: string[] + error: BlockError | null +} + +export interface DAGEdge { + from: string + to: string + inputVariables: string[] +} + +export interface BlockContentDepsDAG { + nodes: DAGNode[] + edges: DAGEdge[] + modulesEdges: DAGEdge[] +} + +// Input block type for the analyzer +export interface AnalyzerBlock { + cellId: string + cell_type: string + source: string + metadata?: Record +} + +// Zod schema for the AST analyzer output +export const AstAnalyzerItemSchema = z.object({ + blockId: z.string(), + definedVariables: z.array(z.string()), + usedVariables: z.array(z.string()), + importedModules: z.array(z.string()).optional(), + error: z + .object({ + type: z.string(), + message: z.string(), + }) + .optional(), +}) + +export const AstAnalyzerSuccessSchema = z.array(AstAnalyzerItemSchema) +export const AstAnalyzerErrorSchema = z.object({ errorMessage: z.string() }) +export const AstAnalyzerResponseSchema = z.union([AstAnalyzerSuccessSchema, AstAnalyzerErrorSchema]) + +export type AstAnalyzerItem = z.infer diff --git a/packages/reactivity/tsconfig.json b/packages/reactivity/tsconfig.json new file mode 100644 index 0000000000..90d76d7e8d --- /dev/null +++ b/packages/reactivity/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 433534a083..b19da6ec1b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,6 +100,12 @@ importers: specifier: 3.25.76 version: 3.25.76 + packages/reactivity: + dependencies: + zod: + specifier: 3.25.76 + version: 3.25.76 + packages: '@babel/generator@7.28.3':

Run in Browser with WebAssembly