From aa04f6614b85fda15d875a3630984367285601ae Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Tue, 11 Nov 2025 15:30:27 +0100 Subject: [PATCH 1/2] feat: bidrectional one shot --- packages/convert/README.md | 72 ++- packages/convert/src/cli.test.ts | 87 ++- packages/convert/src/cli.ts | 39 +- .../convert/src/deepnote-to-jupyter.test.ts | 498 ++++++++++++++++++ packages/convert/src/deepnote-to-jupyter.ts | 170 ++++++ packages/convert/src/index.ts | 2 + packages/convert/src/integration.test.ts | 253 +++++++++ 7 files changed, 1093 insertions(+), 28 deletions(-) create mode 100644 packages/convert/src/deepnote-to-jupyter.test.ts create mode 100644 packages/convert/src/deepnote-to-jupyter.ts create mode 100644 packages/convert/src/integration.test.ts diff --git a/packages/convert/README.md b/packages/convert/README.md index 086d1ed221..7f1a81328c 100644 --- a/packages/convert/README.md +++ b/packages/convert/README.md @@ -1,6 +1,6 @@ # @deepnote/convert -Convert Jupyter Notebook files (`.ipynb`) to Deepnote project files (`.deepnote`). +Bidirectional converter between Jupyter Notebook files (`.ipynb`) and Deepnote project files (`.deepnote`). ## Installation @@ -10,9 +10,11 @@ npm install -g @deepnote/convert ## CLI Usage -The package provides a `deepnote-convert` command-line tool for converting Jupyter notebooks to Deepnote format. +The package provides a `deepnote-convert` command-line tool for bidirectional conversion between Jupyter notebooks and Deepnote projects. -### Convert a Single Notebook +### Jupyter → Deepnote + +#### Convert a Single Notebook Convert a single `.ipynb` file to a `.deepnote` file: @@ -22,7 +24,7 @@ deepnote-convert path/to/notebook.ipynb This will create a `notebook.deepnote` file in the current directory. -### Convert a Directory of Notebooks +#### Convert a Directory of Notebooks Convert all `.ipynb` files in a directory to a single `.deepnote` project: @@ -32,11 +34,21 @@ deepnote-convert path/to/notebooks/ This will create a `notebooks.deepnote` file in the current directory containing all notebooks from the directory. +### Deepnote → Jupyter + +Convert a `.deepnote` project file to Jupyter notebook(s): + +```bash +deepnote-convert path/to/project.deepnote +``` + +This will create a `project/` directory containing `.ipynb` files for each notebook in the project. + ### Options #### `--projectName ` -Set a custom name for the Deepnote project: +Set a custom name for the Deepnote project (Jupyter → Deepnote only): ```bash deepnote-convert notebook.ipynb --projectName "My Analysis" @@ -60,6 +72,8 @@ If not specified, the output file will be saved in the current directory. ### Examples +#### Jupyter → Deepnote + ```bash # Convert a single notebook with custom name deepnote-convert titanic.ipynb --projectName "Titanic Analysis" @@ -71,11 +85,21 @@ deepnote-convert ./analysis --projectName "Data Science Project" -o ./output deepnote-convert ~/notebooks/ml-experiments -o ~/projects/ ``` +#### Deepnote → Jupyter + +```bash +# Convert a Deepnote project to Jupyter notebooks +deepnote-convert project.deepnote + +# Convert with custom output directory +deepnote-convert project.deepnote -o ./output +``` + ## Programmatic Usage -You can also use the conversion function programmatically in your Node.js or TypeScript applications. +You can also use the conversion functions programmatically in your Node.js or TypeScript applications. -### Basic Usage +### Jupyter → Deepnote ```typescript import { convertIpynbFilesToDeepnoteFile } from "@deepnote/convert"; @@ -86,6 +110,40 @@ await convertIpynbFilesToDeepnoteFile(["path/to/notebook.ipynb"], { }); ``` +### Deepnote → Jupyter + +```typescript +import { convertDeepnoteFileToIpynb } from "@deepnote/convert"; + +await convertDeepnoteFileToIpynb("path/to/project.deepnote", { + outputDir: "./output", + addCreatedInDeepnoteCell: true, // Optional, defaults to true +}); +``` + +## Conversion Details + +### Jupyter → Deepnote + +- Code cells → Code blocks +- Markdown cells → Markdown blocks +- Outputs and execution counts are preserved + +### Deepnote → Jupyter + +- **Code blocks** → Code cells (preserved as-is) +- **Markdown blocks** → Markdown cells +- **Text blocks** (h1, h2, h3, p, bullet, todo, callout) → Markdown cells with appropriate formatting +- **SQL blocks** → Code cells with `_dntk.execute_sql()` calls +- **Input blocks** (text, checkbox, select, slider, date, etc.) → Code cells with variable assignments +- **Visualization blocks** → Code cells with visualization specifications +- **Big number blocks** → Code cells with KPI display logic +- **Button blocks** → Code cells with button logic +- **Image blocks** → Markdown cells with `` tags +- **Separator blocks** → Markdown cells with `
` + +Note: Some Deepnote-specific features (like interactivity in input widgets) cannot be fully preserved in standard Jupyter notebooks, but the equivalent Python code is generated to create the same variables and results. + ## License Apache-2.0 diff --git a/packages/convert/src/cli.test.ts b/packages/convert/src/cli.test.ts index 34864fe3d4..0b84f1f36e 100644 --- a/packages/convert/src/cli.test.ts +++ b/packages/convert/src/cli.test.ts @@ -213,17 +213,51 @@ describe('CLI convert function', () => { ).rejects.toThrow('Unsupported file type') }) - it('throws error for .deepnote files', async () => { + it('converts .deepnote files to Jupyter notebooks', async () => { const deepnotePath = path.join(tempDir, 'test.deepnote') - await fs.writeFile(deepnotePath, 'some content', 'utf-8') + const deepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project + name: Test Project + notebooks: + - id: notebook-1 + name: Test Notebook + blocks: + - id: block-1 + type: code + content: 'print("hello")' + sortingKey: a0 + metadata: {} + settings: {} +` + await fs.writeFile(deepnotePath, deepnoteContent, 'utf-8') + + const result = await convert({ + inputPath: deepnotePath, + cwd: tempDir, + }) - // Should throw error - await expect( - convert({ - inputPath: deepnotePath, - cwd: tempDir, - }) - ).rejects.toThrow('.deepnote format is not supported') + // Should create output directory + expect(result).toContain('test') + + // Verify Jupyter notebook was created + const outputDir = result + const notebookPath = path.join(outputDir, 'Test Notebook.ipynb') + const notebookExists = await fs + .access(notebookPath) + .then(() => true) + .catch(() => false) + + expect(notebookExists).toBe(true) + + const notebookContent = await fs.readFile(notebookPath, 'utf-8') + const notebook = JSON.parse(notebookContent) + + expect(notebook.cells).toHaveLength(2) // code cell + "Created in Deepnote" cell + expect(notebook.cells[0].source).toBe('print("hello")') }) it('throws error for non-existent paths', async () => { @@ -411,15 +445,34 @@ describe('CLI convert function', () => { it('correctly handles .deepnote extension with multiple dots in filename', async () => { const deepnotePath = path.join(tempDir, 'my.test.file.deepnote') - await fs.writeFile(deepnotePath, 'some content', 'utf-8') + const deepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project + name: My Test File + notebooks: + - id: notebook-1 + name: Test Notebook + blocks: + - id: block-1 + type: code + content: 'print("test")' + sortingKey: a0 + metadata: {} + settings: {} +` + await fs.writeFile(deepnotePath, deepnoteContent, 'utf-8') + + // Should convert successfully even with multiple dots + const result = await convert({ + inputPath: deepnotePath, + cwd: tempDir, + }) - // Should still throw error for .deepnote files even with multiple dots - await expect( - convert({ - inputPath: deepnotePath, - cwd: tempDir, - }) - ).rejects.toThrow('.deepnote format is not supported') + // Should create output directory with correct name (without extension) + expect(result).toContain('my.test.file') }) it('correctly rejects unsupported files with multiple dots', async () => { diff --git a/packages/convert/src/cli.ts b/packages/convert/src/cli.ts index 07bfc066fb..96e954d0fb 100644 --- a/packages/convert/src/cli.ts +++ b/packages/convert/src/cli.ts @@ -1,8 +1,8 @@ import fs from 'node:fs/promises' -import { basename, extname, resolve } from 'node:path' +import { basename, dirname, extname, resolve } from 'node:path' import chalk from 'chalk' import ora from 'ora' -import { convertIpynbFilesToDeepnoteFile } from '.' +import { convertDeepnoteFileToIpynb, convertIpynbFilesToDeepnoteFile } from '.' interface ConvertOptions { inputPath: string @@ -102,8 +102,39 @@ export async function convert(options: ConvertOptions): Promise { } if (ext === '.deepnote') { - throw new Error('The .deepnote format is not supported for conversion yet.') + const spinner = ora('Converting the Deepnote project to Jupyter Notebooks...').start() + + try { + const filenameWithoutExtension = basename(absolutePath, ext) + + let outputDir: string + if (customOutputPath) { + const absoluteOutputPath = resolve(cwd, customOutputPath) + const stat = await fs.stat(absoluteOutputPath).catch(() => null) + + if (stat?.isDirectory()) { + outputDir = absoluteOutputPath + } else { + // If output path is a file or doesn't exist, use its parent directory + outputDir = dirname(absoluteOutputPath) + } + } else { + // Create a directory with the project name in the current working directory + outputDir = resolve(cwd, filenameWithoutExtension) + } + + await convertDeepnoteFileToIpynb(absolutePath, { outputDir }) + + spinner.succeed(`The Jupyter Notebooks have been saved to ${chalk.bold(outputDir)}`) + + return outputDir + } catch (error) { + spinner.fail('Conversion failed') + throw error + } } - throw new Error('Unsupported file type. Please provide a .ipynb or .deepnote file.') + throw new Error( + 'Unsupported file type. Please provide a .ipynb file, directory of .ipynb files, or a .deepnote file.' + ) } diff --git a/packages/convert/src/deepnote-to-jupyter.test.ts b/packages/convert/src/deepnote-to-jupyter.test.ts new file mode 100644 index 0000000000..391075ce19 --- /dev/null +++ b/packages/convert/src/deepnote-to-jupyter.test.ts @@ -0,0 +1,498 @@ +import fs from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { convertDeepnoteFileToIpynb } from './deepnote-to-jupyter' + +describe('convertDeepnoteFileToIpynb', () => { + const mockOutputDir = '/tmp/test-output' + let writtenFiles: Map + + beforeEach(() => { + writtenFiles = new Map() + + // Mock fs.mkdir + vi.spyOn(fs, 'mkdir').mockResolvedValue(undefined) + + // Mock fs.writeFile to capture what would be written + vi.spyOn(fs, 'writeFile').mockImplementation(async (path, content) => { + writtenFiles.set(path.toString(), content.toString()) + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('converts a simple Deepnote file with code and markdown blocks', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Test Notebook + blocks: + - id: block-1 + type: markdown + content: '# Hello World' + sortingKey: a0 + metadata: {} + - id: block-2 + type: code + content: 'print("Hello")' + sortingKey: a1 + executionCount: 1 + metadata: {} + outputs: + - output_type: stream + name: stdout + text: 'Hello\\n' + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + expect(fs.mkdir).toHaveBeenCalledWith(mockOutputDir, { recursive: true }) + + const outputPath = join(mockOutputDir, 'Test Notebook.ipynb') + expect(writtenFiles.has(outputPath)).toBe(true) + + // biome-ignore lint/style/noNonNullAssertion: Safe in test after verifying file exists + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.nbformat).toBe(4) + expect(notebook.nbformat_minor).toBe(0) + expect(notebook.metadata.deepnote_notebook_id).toBe('notebook-1') + expect(notebook.cells).toHaveLength(2) + + // Check markdown cell + expect(notebook.cells[0].cell_type).toBe('markdown') + expect(notebook.cells[0].source).toBe('# Hello World') + expect(notebook.cells[0].metadata.cell_id).toBe('block-1') + expect(notebook.cells[0].metadata.deepnote_cell_type).toBe('markdown') + + // Check code cell + expect(notebook.cells[1].cell_type).toBe('code') + expect(notebook.cells[1].source).toBe('print("Hello")') + expect(notebook.cells[1].execution_count).toBe(1) + expect(notebook.cells[1].metadata.cell_id).toBe('block-2') + expect(notebook.cells[1].outputs).toEqual([ + { + output_type: 'stream', + name: 'stdout', + text: 'Hello\\n', + }, + ]) + }) + + it('converts input blocks to code cells with variable assignments', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Input Test + blocks: + - id: block-1 + type: input-text + content: '' + sortingKey: a0 + metadata: + deepnote_variable_name: my_input + deepnote_variable_value: 'test value' + - id: block-2 + type: input-checkbox + content: '' + sortingKey: a1 + metadata: + deepnote_variable_name: is_enabled + deepnote_variable_value: true + - id: block-3 + type: input-slider + content: '' + sortingKey: a2 + metadata: + deepnote_variable_name: my_slider + deepnote_variable_value: '5' + deepnote_slider_min_value: 0 + deepnote_slider_max_value: 10 + deepnote_slider_step: 1 + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'Input Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells).toHaveLength(3) + + // Input text + expect(notebook.cells[0].cell_type).toBe('code') + expect(notebook.cells[0].source).toContain('my_input') + expect(notebook.cells[0].source).toContain('test value') + + // Input checkbox + expect(notebook.cells[1].cell_type).toBe('code') + expect(notebook.cells[1].source).toContain('is_enabled') + expect(notebook.cells[1].source).toContain('True') + + // Input slider + expect(notebook.cells[2].cell_type).toBe('code') + expect(notebook.cells[2].source).toContain('my_slider') + expect(notebook.cells[2].source).toContain('5') + }) + + it('converts SQL blocks to code cells with execute_sql calls', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: SQL Test + blocks: + - id: block-1 + type: sql + content: 'SELECT * FROM users' + sortingKey: a0 + metadata: + deepnote_variable_name: df_users + sql_integration_id: sql-integration-123 + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'SQL Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells).toHaveLength(1) + expect(notebook.cells[0].cell_type).toBe('code') + expect(notebook.cells[0].source).toContain('df_users') + expect(notebook.cells[0].source).toContain('_dntk.execute_sql') + expect(notebook.cells[0].source).toContain('SELECT * FROM users') + }) + + it('converts text blocks to markdown cells', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Text Test + blocks: + - id: block-1 + type: text-cell-h1 + content: 'Main Title' + sortingKey: a0 + metadata: {} + - id: block-2 + type: text-cell-h2 + content: 'Subtitle' + sortingKey: a1 + metadata: {} + - id: block-3 + type: text-cell-p + content: 'A paragraph of text' + sortingKey: a2 + metadata: {} + - id: block-4 + type: text-cell-bullet + content: 'Bullet point' + sortingKey: a3 + metadata: {} + - id: block-5 + type: text-cell-todo + content: 'Todo item' + sortingKey: a4 + metadata: + checked: true + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'Text Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells).toHaveLength(5) + + // H1 + expect(notebook.cells[0].cell_type).toBe('markdown') + expect(notebook.cells[0].source).toBe('# Main Title') + + // H2 + expect(notebook.cells[1].cell_type).toBe('markdown') + expect(notebook.cells[1].source).toBe('## Subtitle') + + // Paragraph + expect(notebook.cells[2].cell_type).toBe('markdown') + expect(notebook.cells[2].source).toBe('A paragraph of text') + + // Bullet + expect(notebook.cells[3].cell_type).toBe('markdown') + expect(notebook.cells[3].source).toBe('- Bullet point') + + // Todo + expect(notebook.cells[4].cell_type).toBe('markdown') + expect(notebook.cells[4].source).toBe('- [x] Todo item') + }) + + it('converts separator and image blocks to markdown', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Media Test + blocks: + - id: block-1 + type: separator + content: '' + sortingKey: a0 + metadata: {} + - id: block-2 + type: image + content: '' + sortingKey: a1 + metadata: + deepnote_img_src: 'https://example.com/image.png' + deepnote_img_width: '500' + deepnote_img_alignment: 'center' + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'Media Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells).toHaveLength(2) + + // Separator + expect(notebook.cells[0].cell_type).toBe('markdown') + expect(notebook.cells[0].source).toBe('
') + + // Image + expect(notebook.cells[1].cell_type).toBe('markdown') + expect(notebook.cells[1].source).toContain('img src') + expect(notebook.cells[1].source).toContain('https://example.com/image.png') + expect(notebook.cells[1].source).toContain('width="500"') + }) + + it('adds "Created in Deepnote" cell when enabled', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-123 + name: Test Project + notebooks: + - id: notebook-1 + name: Test + blocks: + - id: block-1 + type: code + content: 'print("test")' + sortingKey: a0 + metadata: {} + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: true, + }) + + const outputPath = join(mockOutputDir, 'Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells).toHaveLength(2) + + const lastCell = notebook.cells[1] + expect(lastCell.cell_type).toBe('markdown') + expect(lastCell.source).toContain('Created in') + expect(lastCell.source).toContain('Deepnote') + expect(lastCell.source).toContain('test-project-123') + expect(lastCell.metadata.created_in_deepnote_cell).toBe(true) + }) + + it('converts multiple notebooks in a project', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: First Notebook + blocks: + - id: block-1 + type: code + content: 'print("first")' + sortingKey: a0 + metadata: {} + - id: notebook-2 + name: Second Notebook + blocks: + - id: block-2 + type: code + content: 'print("second")' + sortingKey: a0 + metadata: {} + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + expect(writtenFiles.size).toBe(2) + + const firstPath = join(mockOutputDir, 'First Notebook.ipynb') + const secondPath = join(mockOutputDir, 'Second Notebook.ipynb') + + expect(writtenFiles.has(firstPath)).toBe(true) + expect(writtenFiles.has(secondPath)).toBe(true) + + // biome-ignore lint/style/noNonNullAssertion: Safe in test after verifying files exist + const firstNotebook = JSON.parse(writtenFiles.get(firstPath)!) + // biome-ignore lint/style/noNonNullAssertion: Safe in test after verifying files exist + const secondNotebook = JSON.parse(writtenFiles.get(secondPath)!) + + expect(firstNotebook.cells[0].source).toBe('print("first")') + expect(secondNotebook.cells[0].source).toBe('print("second")') + }) + + it('handles blocks without outputs gracefully', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Test + blocks: + - id: block-1 + type: code + content: 'x = 1' + sortingKey: a0 + metadata: {} + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells[0].outputs).toEqual([]) + expect(notebook.cells[0].execution_count).toBe(null) + }) + + it('handles date range input blocks', async () => { + const mockDeepnoteContent = ` +version: 1.0.0 +metadata: + createdAt: '2025-01-01T00:00:00Z' +project: + id: test-project-id + name: Test Project + notebooks: + - id: notebook-1 + name: Test + blocks: + - id: block-1 + type: input-date-range + content: '' + sortingKey: a0 + metadata: + deepnote_variable_name: date_range + deepnote_variable_value: + - '2025-01-01' + - '2025-01-31' + settings: {} +` + + vi.spyOn(fs, 'readFile').mockResolvedValue(mockDeepnoteContent) + + await convertDeepnoteFileToIpynb('test.deepnote', { + outputDir: mockOutputDir, + addCreatedInDeepnoteCell: false, + }) + + const outputPath = join(mockOutputDir, 'Test.ipynb') + // biome-ignore lint/style/noNonNullAssertion: Safe in test after conversion + const notebook = JSON.parse(writtenFiles.get(outputPath)!) + + expect(notebook.cells[0].cell_type).toBe('code') + expect(notebook.cells[0].source).toContain('date_range') + expect(notebook.cells[0].source).toContain('2025') + }) +}) diff --git a/packages/convert/src/deepnote-to-jupyter.ts b/packages/convert/src/deepnote-to-jupyter.ts new file mode 100644 index 0000000000..ced33cc436 --- /dev/null +++ b/packages/convert/src/deepnote-to-jupyter.ts @@ -0,0 +1,170 @@ +import fs from 'node:fs/promises' +import { join } from 'node:path' +import type { DeepnoteBlock } from '@deepnote/blocks' +import { createMarkdown, createPythonCode, deserializeDeepnoteFile } from '@deepnote/blocks' + +interface JupyterCell { + cell_type: 'code' | 'markdown' + metadata: Record + source: string + execution_count?: number | null + // biome-ignore lint/suspicious/noExplicitAny: Jupyter outputs can have various types + outputs?: any[] +} + +interface JupyterNotebook { + cells: JupyterCell[] + metadata: { + deepnote_notebook_id?: string + deepnote_execution_queue?: unknown[] + [key: string]: unknown + } + nbformat: number + nbformat_minor: number +} + +export interface ConvertDeepnoteFileToIpynbOptions { + outputDir: string + addCreatedInDeepnoteCell?: boolean +} + +const CREATED_IN_DEEPNOTE_SOURCE = ` +Created in deepnote.com +Created in Deepnote` + +function convertBlockToJupyterCell(block: DeepnoteBlock): JupyterCell { + const metadata: Record = { + ...block.metadata, + cell_id: block.id, + deepnote_cell_type: block.type, + } + + // Determine if this should be a code cell or markdown cell + const isCodeBlock = shouldConvertToCodeCell(block) + + if (isCodeBlock) { + let source: string + + // For code blocks, use the original source + if (block.type === 'code') { + source = block.content ?? '' + } else { + // For other executable blocks (SQL, input, visualization, etc.), + // generate the equivalent Python code + try { + source = createPythonCode(block) + } catch (error) { + // If we can't generate Python code, fall back to a comment + const message = error instanceof Error ? error.message : String(error) + source = `# Unable to convert ${block.type} block: ${message}` + } + } + + return { + cell_type: 'code', + metadata, + source, + execution_count: block.executionCount ?? null, + outputs: block.outputs ?? [], + } + } + + // Convert to markdown cell + let source: string + + if (block.type === 'markdown') { + source = block.content ?? '' + } else { + // For text blocks, images, separators, etc., use createMarkdown + try { + source = createMarkdown(block) + } catch (_error) { + // If we can't generate markdown, use the content directly + source = block.content ?? '' + } + } + + return { + cell_type: 'markdown', + metadata, + source, + } +} + +function shouldConvertToCodeCell(block: DeepnoteBlock): boolean { + const codeBlockTypes = [ + 'code', + 'sql', + 'input-text', + 'input-textarea', + 'input-checkbox', + 'input-select', + 'input-slider', + 'input-date', + 'input-date-range', + 'input-file', + 'visualization', + 'big-number', + 'button', + ] + + return codeBlockTypes.includes(block.type) +} + +function createCreatedInDeepnoteCell(projectId: string): JupyterCell { + return { + cell_type: 'markdown', + metadata: { + created_in_deepnote_cell: true, + deepnote_cell_type: 'markdown', + }, + source: CREATED_IN_DEEPNOTE_SOURCE.replace('{projectId}', projectId), + } +} + +/** + * Converts a Deepnote project file (.deepnote) to Jupyter notebook files (.ipynb). + */ +export async function convertDeepnoteFileToIpynb( + deepnoteFilePath: string, + options: ConvertDeepnoteFileToIpynbOptions +): Promise { + // Read and parse the .deepnote file + const yamlContent = await fs.readFile(deepnoteFilePath, 'utf-8') + const deepnoteFile = deserializeDeepnoteFile(yamlContent) + + // Create output directory + await fs.mkdir(options.outputDir, { recursive: true }) + + const addCreatedInDeepnoteCell = options.addCreatedInDeepnoteCell ?? true + + // Convert each notebook in the project + for (const notebook of deepnoteFile.project.notebooks) { + const jupyterNotebook: JupyterNotebook = { + cells: [], + metadata: { + deepnote_notebook_id: notebook.id, + deepnote_execution_queue: [], + }, + nbformat: 4, + nbformat_minor: 0, + } + + // Convert each block to a Jupyter cell + for (const block of notebook.blocks) { + const cell = convertBlockToJupyterCell(block) + jupyterNotebook.cells.push(cell) + } + + // Optionally add "Created in Deepnote" cell + if (addCreatedInDeepnoteCell) { + jupyterNotebook.cells.push(createCreatedInDeepnoteCell(deepnoteFile.project.id)) + } + + // Write the Jupyter notebook file + const outputFileName = `${notebook.name}.ipynb` + const outputPath = join(options.outputDir, outputFileName) + + await fs.writeFile(outputPath, JSON.stringify(jupyterNotebook, null, 2), 'utf-8') + } +} diff --git a/packages/convert/src/index.ts b/packages/convert/src/index.ts index 86388db60f..a44eccef47 100644 --- a/packages/convert/src/index.ts +++ b/packages/convert/src/index.ts @@ -1,2 +1,4 @@ +export type { ConvertDeepnoteFileToIpynbOptions } from './deepnote-to-jupyter' +export { convertDeepnoteFileToIpynb } from './deepnote-to-jupyter' export type { ConvertIpynbFilesToDeepnoteFileOptions } from './jupyter-to-deepnote' export { convertIpynbFilesToDeepnoteFile } from './jupyter-to-deepnote' diff --git a/packages/convert/src/integration.test.ts b/packages/convert/src/integration.test.ts new file mode 100644 index 0000000000..1e2ee98100 --- /dev/null +++ b/packages/convert/src/integration.test.ts @@ -0,0 +1,253 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { describe, expect, it } from 'vitest' +import { convertDeepnoteFileToIpynb } from './deepnote-to-jupyter' +import { convertIpynbFilesToDeepnoteFile } from './jupyter-to-deepnote' + +describe('Integration tests: bidirectional conversion', () => { + const examplesDir = path.resolve(__dirname, '../../../examples') + + it('converts 1_hello_world.deepnote to ipynb and back', async () => { + const originalDeepnotePath = path.join(examplesDir, '1_hello_world.deepnote') + const tempDir = path.join(__dirname, '../../../tmp/integration-test-hello-world') + + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + await fs.mkdir(tempDir, { recursive: true }) + + try { + // 1. Read original .deepnote file + const originalDeepnoteContent = await fs.readFile(originalDeepnotePath, 'utf-8') + + // 2. Convert to Jupyter + const ipynbDir = path.join(tempDir, 'ipynb-output') + await convertDeepnoteFileToIpynb(originalDeepnotePath, { + outputDir: ipynbDir, + addCreatedInDeepnoteCell: false, // Don't add extra cell for clean comparison + }) + + // 3. Read generated Jupyter notebook(s) + const ipynbFiles = await fs.readdir(ipynbDir) + expect(ipynbFiles.length).toBeGreaterThan(0) + expect(ipynbFiles.some(f => f.endsWith('.ipynb'))).toBe(true) + + // 4. Convert back to .deepnote + const roundtripDeepnotePath = path.join(tempDir, 'roundtrip.deepnote') + const ipynbPaths = ipynbFiles.filter(f => f.endsWith('.ipynb')).map(f => path.join(ipynbDir, f)) + + await convertIpynbFilesToDeepnoteFile(ipynbPaths, { + projectName: 'Hello World', + outputPath: roundtripDeepnotePath, + }) + + // 5. Read roundtrip .deepnote file + const roundtripDeepnoteContent = await fs.readFile(roundtripDeepnotePath, 'utf-8') + + // 6. Parse both files for comparison + const _originalLines = originalDeepnoteContent.split('\n').filter(line => { + // Filter out metadata that changes (timestamps, IDs) + return ( + !line.includes('createdAt:') && + !line.includes('modifiedAt:') && + !line.includes('exportedAt:') && + !line.includes('id:') && + !line.includes('blockGroup:') && + !line.includes('execution_start:') && + !line.includes('execution_millis:') && + !line.includes('execution_context_id:') + ) + }) + + const roundtripLines = roundtripDeepnoteContent.split('\n').filter(line => { + return ( + !line.includes('createdAt:') && + !line.includes('modifiedAt:') && + !line.includes('exportedAt:') && + !line.includes('id:') && + !line.includes('blockGroup:') && + !line.includes('execution_start:') && + !line.includes('execution_millis:') && + !line.includes('execution_context_id:') + ) + }) + + // Compare content (should be similar after filtering dynamic fields) + expect(roundtripLines.length).toBeGreaterThan(0) + + // Check key content is preserved + expect(roundtripDeepnoteContent).toContain('print("Hello world!")') + expect(roundtripDeepnoteContent).toContain('type: code') + } finally { + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) + + it('converts 2_blocks.deepnote to ipynb and back preserving block types', async () => { + const originalDeepnotePath = path.join(examplesDir, '2_blocks.deepnote') + const tempDir = path.join(__dirname, '../../../tmp/integration-test-blocks') + + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + await fs.mkdir(tempDir, { recursive: true }) + + try { + // 1. Read original .deepnote file + const originalDeepnoteContent = await fs.readFile(originalDeepnotePath, 'utf-8') + + // 2. Convert to Jupyter + const ipynbDir = path.join(tempDir, 'ipynb-output') + await convertDeepnoteFileToIpynb(originalDeepnotePath, { + outputDir: ipynbDir, + addCreatedInDeepnoteCell: false, + }) + + // 3. Verify Jupyter notebooks were created + const ipynbFiles = await fs.readdir(ipynbDir) + const ipynbPaths = ipynbFiles.filter(f => f.endsWith('.ipynb')).map(f => path.join(ipynbDir, f)) + + expect(ipynbPaths.length).toBeGreaterThan(0) + + // 4. Check that various block types are converted + for (const ipynbPath of ipynbPaths) { + const ipynbContent = await fs.readFile(ipynbPath, 'utf-8') + const notebook = JSON.parse(ipynbContent) + + // Verify cells exist + expect(notebook.cells.length).toBeGreaterThan(0) + + // Check that metadata is preserved + for (const cell of notebook.cells) { + expect(cell.metadata).toBeDefined() + expect(cell.metadata.deepnote_cell_type).toBeDefined() + } + } + + // 5. Convert back to .deepnote + const roundtripDeepnotePath = path.join(tempDir, 'roundtrip.deepnote') + + await convertIpynbFilesToDeepnoteFile(ipynbPaths, { + projectName: 'blocks.deepnote', + outputPath: roundtripDeepnotePath, + }) + + // 6. Verify roundtrip file has expected content + const roundtripDeepnoteContent = await fs.readFile(roundtripDeepnotePath, 'utf-8') + + // Check that key content from input blocks is preserved + expect(roundtripDeepnoteContent).toContain('markdown') + expect(roundtripDeepnoteContent).toContain('code') + + // Input blocks get converted to code (variable assignments), so verify the variables exist + expect(originalDeepnoteContent).toContain('input_text') + expect(originalDeepnoteContent).toContain('input_checkbox') + expect(originalDeepnoteContent).toContain('input_select') + } finally { + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) + + it('preserves code cell outputs through roundtrip conversion', async () => { + const originalDeepnotePath = path.join(examplesDir, '1_hello_world.deepnote') + const tempDir = path.join(__dirname, '../../../tmp/integration-test-outputs') + + await fs.rm(tempDir, { recursive: true, force: true }) + await fs.mkdir(tempDir, { recursive: true }) + + try { + // Convert to Jupyter + const ipynbDir = path.join(tempDir, 'ipynb-output') + await convertDeepnoteFileToIpynb(originalDeepnotePath, { + outputDir: ipynbDir, + addCreatedInDeepnoteCell: false, + }) + + // Read the Jupyter notebook + const ipynbFiles = await fs.readdir(ipynbDir) + const firstIpynb = ipynbFiles.find(f => f.endsWith('.ipynb')) + expect(firstIpynb).toBeDefined() + + // biome-ignore lint/style/noNonNullAssertion: Safe after checking with expect + const ipynbPath = path.join(ipynbDir, firstIpynb!) + const ipynbContent = await fs.readFile(ipynbPath, 'utf-8') + const notebook = JSON.parse(ipynbContent) + + // Find code cells with outputs + const codeCellsWithOutputs = notebook.cells.filter( + // biome-ignore lint/suspicious/noExplicitAny: Jupyter notebook format is flexible + (cell: any) => cell.cell_type === 'code' && cell.outputs && cell.outputs.length > 0 + ) + + expect(codeCellsWithOutputs.length).toBeGreaterThan(0) + + // Convert back to Deepnote + const roundtripPath = path.join(tempDir, 'roundtrip.deepnote') + await convertIpynbFilesToDeepnoteFile([ipynbPath], { + projectName: 'Test', + outputPath: roundtripPath, + }) + + // Verify outputs are preserved + const roundtripContent = await fs.readFile(roundtripPath, 'utf-8') + expect(roundtripContent).toContain('outputs:') + expect(roundtripContent).toContain('Hello world!') + } finally { + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) + + it('handles markdown content correctly in roundtrip', async () => { + const originalDeepnotePath = path.join(examplesDir, '2_blocks.deepnote') + const tempDir = path.join(__dirname, '../../../tmp/integration-test-markdown') + + await fs.rm(tempDir, { recursive: true, force: true }) + await fs.mkdir(tempDir, { recursive: true }) + + try { + // Read original to check markdown content + const originalContent = await fs.readFile(originalDeepnotePath, 'utf-8') + + // Convert to Jupyter + const ipynbDir = path.join(tempDir, 'ipynb-output') + await convertDeepnoteFileToIpynb(originalDeepnotePath, { + outputDir: ipynbDir, + addCreatedInDeepnoteCell: false, + }) + + // Check that markdown is present in Jupyter format + const ipynbFiles = await fs.readdir(ipynbDir) + const ipynbPaths = ipynbFiles.filter(f => f.endsWith('.ipynb')).map(f => path.join(ipynbDir, f)) + + for (const ipynbPath of ipynbPaths) { + const ipynbContent = await fs.readFile(ipynbPath, 'utf-8') + const notebook = JSON.parse(ipynbContent) + + // biome-ignore lint/suspicious/noExplicitAny: Jupyter notebook format is flexible + const markdownCells = notebook.cells.filter((cell: any) => cell.cell_type === 'markdown') + + if (markdownCells.length > 0) { + // Verify markdown cells have content + expect(markdownCells[0].source).toBeDefined() + expect(markdownCells[0].source.length).toBeGreaterThan(0) + } + } + + // Convert back + const roundtripPath = path.join(tempDir, 'roundtrip.deepnote') + await convertIpynbFilesToDeepnoteFile(ipynbPaths, { + projectName: 'Test', + outputPath: roundtripPath, + }) + + const roundtripContent = await fs.readFile(roundtripPath, 'utf-8') + + // Check markdown is preserved (though it might be in a different block type) + if (originalContent.includes('# This is a markdown heading')) { + expect(roundtripContent).toContain('This is a markdown heading') + } + } finally { + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) +}) From 7281a1f8219029e95e0edfd2d2f3ec967cb88393 Mon Sep 17 00:00:00 2001 From: James Hobbs Date: Tue, 11 Nov 2025 15:41:08 +0100 Subject: [PATCH 2/2] feat: preserve ids --- packages/convert/src/deepnote-to-jupyter.ts | 13 +++ packages/convert/src/integration.test.ts | 87 +++++++++++++++++++ .../convert/src/jupyter-to-deepnote.test.ts | 57 ++++++++++-- packages/convert/src/jupyter-to-deepnote.ts | 46 ++++++++-- 4 files changed, 190 insertions(+), 13 deletions(-) diff --git a/packages/convert/src/deepnote-to-jupyter.ts b/packages/convert/src/deepnote-to-jupyter.ts index ced33cc436..9944c8edaf 100644 --- a/packages/convert/src/deepnote-to-jupyter.ts +++ b/packages/convert/src/deepnote-to-jupyter.ts @@ -17,6 +17,10 @@ interface JupyterNotebook { metadata: { deepnote_notebook_id?: string deepnote_execution_queue?: unknown[] + deepnote?: { + original_project_id?: string + original_notebook_id?: string + } [key: string]: unknown } nbformat: number @@ -37,6 +41,11 @@ function convertBlockToJupyterCell(block: DeepnoteBlock): JupyterCell { ...block.metadata, cell_id: block.id, deepnote_cell_type: block.type, + deepnote_to_be_reused: { + block_id: block.id, + block_group: block.blockGroup, + sorting_key: block.sortingKey, + }, } // Determine if this should be a code cell or markdown cell @@ -145,6 +154,10 @@ export async function convertDeepnoteFileToIpynb( metadata: { deepnote_notebook_id: notebook.id, deepnote_execution_queue: [], + deepnote: { + original_project_id: deepnoteFile.project.id, + original_notebook_id: notebook.id, + }, }, nbformat: 4, nbformat_minor: 0, diff --git a/packages/convert/src/integration.test.ts b/packages/convert/src/integration.test.ts index 1e2ee98100..7a90ab8882 100644 --- a/packages/convert/src/integration.test.ts +++ b/packages/convert/src/integration.test.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises' import path from 'node:path' import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' import { convertDeepnoteFileToIpynb } from './deepnote-to-jupyter' import { convertIpynbFilesToDeepnoteFile } from './jupyter-to-deepnote' @@ -250,4 +251,90 @@ describe('Integration tests: bidirectional conversion', () => { await fs.rm(tempDir, { recursive: true, force: true }) } }) + + it('preserves IDs through roundtrip conversion', async () => { + const originalDeepnotePath = path.join(examplesDir, '1_hello_world.deepnote') + const tempDir = path.join(__dirname, '../../../tmp/integration-test-id-preservation') + + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + await fs.mkdir(tempDir, { recursive: true }) + + try { + // 1. Parse original .deepnote file to extract IDs + const originalDeepnoteContent = await fs.readFile(originalDeepnotePath, 'utf-8') + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + const originalDeepnote = parse(originalDeepnoteContent) as any + + const originalProjectId = originalDeepnote.project.id + const originalNotebookId = originalDeepnote.project.notebooks[0].id + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + const originalBlockIds = originalDeepnote.project.notebooks[0].blocks.map((b: any) => b.id) + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + const originalBlockGroups = originalDeepnote.project.notebooks[0].blocks.map((b: any) => b.blockGroup) + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + const originalSortingKeys = originalDeepnote.project.notebooks[0].blocks.map((b: any) => b.sortingKey) + + // 2. Convert to Jupyter + const ipynbDir = path.join(tempDir, 'ipynb-output') + await convertDeepnoteFileToIpynb(originalDeepnotePath, { + outputDir: ipynbDir, + addCreatedInDeepnoteCell: false, + }) + + // 3. Read Jupyter notebook and verify metadata is stored + const ipynbFiles = await fs.readdir(ipynbDir) + const ipynbPath = path.join(ipynbDir, ipynbFiles[0]) + const ipynbContent = await fs.readFile(ipynbPath, 'utf-8') + const ipynb = JSON.parse(ipynbContent) + + // Verify metadata is stored in Jupyter notebook + expect(ipynb.metadata.deepnote.original_project_id).toBe(originalProjectId) + expect(ipynb.metadata.deepnote.original_notebook_id).toBe(originalNotebookId) + expect(ipynb.cells[0].metadata.deepnote_to_be_reused.block_id).toBe(originalBlockIds[0]) + expect(ipynb.cells[0].metadata.deepnote_to_be_reused.block_group).toBe(originalBlockGroups[0]) + expect(ipynb.cells[0].metadata.deepnote_to_be_reused.sorting_key).toBe(originalSortingKeys[0]) + + // 4. Convert back to .deepnote + const roundtripDeepnotePath = path.join(tempDir, 'roundtrip.deepnote') + await convertIpynbFilesToDeepnoteFile([ipynbPath], { + projectName: 'Hello World', + outputPath: roundtripDeepnotePath, + }) + + // 5. Parse roundtrip file and verify IDs are preserved + const roundtripDeepnoteContent = await fs.readFile(roundtripDeepnotePath, 'utf-8') + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + const roundtripDeepnote = parse(roundtripDeepnoteContent) as any + + // Verify all IDs are preserved + expect(roundtripDeepnote.project.id).toBe(originalProjectId) + expect(roundtripDeepnote.project.notebooks[0].id).toBe(originalNotebookId) + expect(roundtripDeepnote.project.notebooks[0].blocks[0].id).toBe(originalBlockIds[0]) + expect(roundtripDeepnote.project.notebooks[0].blocks[0].blockGroup).toBe(originalBlockGroups[0]) + expect(roundtripDeepnote.project.notebooks[0].blocks[0].sortingKey).toBe(originalSortingKeys[0]) + + // Verify all original blocks' IDs are preserved (excluding any new cells like "Created in Deepnote") + const roundtripBlockIds = roundtripDeepnote.project.notebooks[0].blocks + .slice(0, originalBlockIds.length) + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + .map((b: any) => b.id) + expect(roundtripBlockIds).toEqual(originalBlockIds) + + const roundtripBlockGroups = roundtripDeepnote.project.notebooks[0].blocks + .slice(0, originalBlockGroups.length) + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + .map((b: any) => b.blockGroup) + expect(roundtripBlockGroups).toEqual(originalBlockGroups) + + const roundtripSortingKeys = roundtripDeepnote.project.notebooks[0].blocks + .slice(0, originalSortingKeys.length) + // biome-ignore lint/suspicious/noExplicitAny: Deepnote file structure is flexible + .map((b: any) => b.sortingKey) + expect(roundtripSortingKeys).toEqual(originalSortingKeys) + } finally { + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) }) diff --git a/packages/convert/src/jupyter-to-deepnote.test.ts b/packages/convert/src/jupyter-to-deepnote.test.ts index 64a871f52a..2e33410520 100644 --- a/packages/convert/src/jupyter-to-deepnote.test.ts +++ b/packages/convert/src/jupyter-to-deepnote.test.ts @@ -875,7 +875,15 @@ describe('snapshot tests - exact YAML output format', () => { # Any results you write to the current directory are saved as output. id: test-uuid-007 - metadata: {} + metadata: + _kg_hide-input: false + execution: + iopub.execute_input: 2025-09-01T09:06:57.140357Z + iopub.status.busy: 2025-09-01T09:06:57.139933Z + iopub.status.idle: 2025-09-01T09:06:57.371877Z + shell.execute_reply: 2025-09-01T09:06:57.371054Z + shell.execute_reply.started: 2025-09-01T09:06:57.140287Z + trusted: true outputs: [] sortingKey: "2" type: code @@ -909,7 +917,14 @@ describe('snapshot tests - exact YAML output format', () => { train_data = pd.read_csv("/kaggle/input/titanic/train.csv") train_data.head() id: test-uuid-011 - metadata: {} + metadata: + execution: + iopub.execute_input: 2025-09-01T09:06:57.374629Z + iopub.status.busy: 2025-09-01T09:06:57.374278Z + iopub.status.idle: 2025-09-01T09:06:57.427646Z + shell.execute_reply: 2025-09-01T09:06:57.426732Z + shell.execute_reply.started: 2025-09-01T09:06:57.374555Z + trusted: true outputs: [] sortingKey: "4" type: code @@ -958,7 +973,14 @@ describe('snapshot tests - exact YAML output format', () => { test_data = pd.read_csv("/kaggle/input/titanic/test.csv") test_data.head() id: test-uuid-015 - metadata: {} + metadata: + execution: + iopub.execute_input: 2025-09-01T09:06:57.431176Z + iopub.status.busy: 2025-09-01T09:06:57.430967Z + iopub.status.idle: 2025-09-01T09:06:57.470048Z + shell.execute_reply: 2025-09-01T09:06:57.468935Z + shell.execute_reply.started: 2025-09-01T09:06:57.431137Z + trusted: true outputs: [] sortingKey: "6" type: code @@ -1012,7 +1034,15 @@ describe('snapshot tests - exact YAML output format', () => { print("% of women who survived:", rate_women) id: test-uuid-019 - metadata: {} + metadata: + execution: + iopub.execute_input: 2025-09-01T09:06:57.473657Z + iopub.status.busy: 2025-09-01T09:06:57.473144Z + iopub.status.idle: 2025-09-01T09:06:57.484251Z + shell.execute_reply: 2025-09-01T09:06:57.483288Z + shell.execute_reply.started: 2025-09-01T09:06:57.473544Z + scrolled: true + trusted: true outputs: [] sortingKey: "8" type: code @@ -1037,7 +1067,14 @@ describe('snapshot tests - exact YAML output format', () => { print("% of men who survived:", rate_men) id: test-uuid-023 - metadata: {} + metadata: + execution: + iopub.execute_input: 2025-09-01T09:06:57.486507Z + iopub.status.busy: 2025-09-01T09:06:57.486162Z + iopub.status.idle: 2025-09-01T09:06:57.506208Z + shell.execute_reply: 2025-09-01T09:06:57.505447Z + shell.execute_reply.started: 2025-09-01T09:06:57.486442Z + trusted: true outputs: [] sortingKey: a type: code @@ -1123,7 +1160,15 @@ describe('snapshot tests - exact YAML output format', () => { print("Your submission was successfully saved!") id: test-uuid-027 - metadata: {} + metadata: + _kg_hide-output: false + execution: + iopub.execute_input: 2025-09-01T09:06:57.507870Z + iopub.status.busy: 2025-09-01T09:06:57.507569Z + iopub.status.idle: 2025-09-01T09:07:00.901413Z + shell.execute_reply: 2025-09-01T09:07:00.900466Z + shell.execute_reply.started: 2025-09-01T09:06:57.507805Z + trusted: true outputs: [] sortingKey: c type: code diff --git a/packages/convert/src/jupyter-to-deepnote.ts b/packages/convert/src/jupyter-to-deepnote.ts index 7031344aa2..8068fc172f 100644 --- a/packages/convert/src/jupyter-to-deepnote.ts +++ b/packages/convert/src/jupyter-to-deepnote.ts @@ -18,7 +18,13 @@ interface IpynbFile { outputs: any[] source: string | string[] }[] - metadata: Record + metadata: { + deepnote?: { + original_project_id?: string + original_notebook_id?: string + } + [key: string]: unknown + } nbformat: number nbformat_minor: number } @@ -30,12 +36,23 @@ export async function convertIpynbFilesToDeepnoteFile( inputFilePaths: string[], options: ConvertIpynbFilesToDeepnoteFileOptions ): Promise { + // Try to get original project ID from first notebook if it exists + let originalProjectId: string | undefined + if (inputFilePaths.length > 0) { + try { + const firstIpynb = await parseIpynbFile(inputFilePaths[0]) + originalProjectId = firstIpynb.metadata?.deepnote?.original_project_id as string | undefined + } catch { + // Ignore errors, we'll just use a new ID + } + } + const deepnoteFile: DeepnoteFile = { metadata: { createdAt: new Date().toISOString(), }, project: { - id: v4(), + id: originalProjectId ?? v4(), initNotebookId: undefined, integrations: [], name: options.projectName, @@ -54,14 +71,26 @@ export async function convertIpynbFilesToDeepnoteFile( const blocks = ipynb.cells.map((cell, index) => { const source = Array.isArray(cell.source) ? cell.source.join('') : cell.source + // Check if cell has preserved Deepnote metadata + const deepnoteMetadata = cell.metadata?.deepnote_to_be_reused as + | { + block_id?: string + block_group?: string + sorting_key?: string + } + | undefined + + // Filter out deepnote_to_be_reused from metadata when copying + const { deepnote_to_be_reused: _, deepnote_cell_type: __, cell_id: ___, ...restMetadata } = cell.metadata || {} + const block = { - blockGroup: v4(), + blockGroup: deepnoteMetadata?.block_group ?? v4(), content: source, executionCount: cell.execution_count ?? undefined, - id: v4(), - metadata: {}, + id: deepnoteMetadata?.block_id ?? v4(), + metadata: restMetadata, outputs: cell.cell_type === 'code' ? cell.outputs : undefined, - sortingKey: createSortingKey(index), + sortingKey: deepnoteMetadata?.sorting_key ?? createSortingKey(index), type: cell.cell_type === 'code' ? 'code' : 'markdown', version: 1, } @@ -69,10 +98,13 @@ export async function convertIpynbFilesToDeepnoteFile( return block }) + // Check if notebook has preserved ID + const originalNotebookId = ipynb.metadata?.deepnote?.original_notebook_id as string | undefined + deepnoteFile.project.notebooks.push({ blocks, executionMode: 'block', - id: v4(), + id: originalNotebookId ?? v4(), isModule: false, name, workingDirectory: undefined,