Add command line nb2py to convert a notebook into a python example (#16)

xadupre · web-flow · commit b853b3246583 · 2023-07-24T01:52:24.000+02:00
* Add command nb2rst

* add unit test for command line

* rename a command

* complete rename

* action

* fix ut

* unittests

* update CHANGELOGS
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -25,6 +25,9 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Install pandoc
+        run: sudo apt-get install pandoc
+
       - name: Install requirements
         run: python -m pip install -r requirements.txt
 
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,8 @@ _doc/examples/plot_*.png
 _doc/_static/require.js
 _doc/_static/viz.js
 _unittests/ut__main/*.png
+_unittests/ut__main/data/*.rst
+_unittests/ut__main/data/*.py
 _doc/examples/*.html
 _unittests/ut__main/_cache/*
 _unittests/ut__main/*.html
diff --git a/CHANGELOGS.rst b/CHANGELOGS.rst
@@ -4,6 +4,8 @@ Change Logs
 0.2.0
 +++++
 
+* :pr:`16`: add command line nb2py to convert notebook to python files
+* :pr:`11`: add function make_linkcode_resolve
 * :pr:`6`: add sphinx extension gdot
 * :pr:`5`: add sphinx extension docassert
 * :pr:`4`: add sphinx extension mathdef, exref, blocref
diff --git a/_doc/api/index.rst b/_doc/api/index.rst
@@ -16,6 +16,7 @@ Extensions
     epkg
     gdot
     runpython
+    tools
 
 Helpers
 =======
diff --git a/_doc/api/tools.rst b/_doc/api/tools.rst
@@ -0,0 +1,13 @@
+=====
+tools
+=====
+
+Convert notebooks into examples
+===============================
+
+The command line converts every notebook in a folder
+into exmaples which can be used into a sphinx gallery.
+
+::
+
+    python -m sphinx_runpython --help
diff --git a/_doc/conf.py b/_doc/conf.py
@@ -84,6 +84,7 @@
     "pandoc": "https://johnmacfarlane.net/pandoc/",
     "Pandoc": "https://johnmacfarlane.net/pandoc/",
     "PNG": "https://en.wikipedia.org/wiki/PNG",
+    "pypandoc": "https://github.com/JessicaTegner/pypandoc",
     "python": "https://www.python.org/",
     "RST": "https://fr.wikipedia.org/wiki/ReStructuredText",
     "sphinx": "https://www.sphinx-doc.org/en/master/",
diff --git a/_unittests/ut__main/data/float_and_double_rouding.ipynb b/_unittests/ut__main/data/float_and_double_rouding.ipynb
diff --git a/_unittests/ut__main/test_cmd.py b/_unittests/ut__main/test_cmd.py
@@ -0,0 +1,25 @@
+import platform
+import unittest
+import os
+from sphinx_runpython.ext_test_case import ExtTestCase
+from sphinx_runpython._cmd_helper import get_parser, nb2py
+
+
+class TestCmd(ExtTestCase):
+    def test_cmd(self):
+        parser = get_parser()
+        self.assertNotEmpty(parser)
+
+    @unittest.skipIf(platform.system() != "Linux", reason="pandoc not installed")
+    def test_convert(self):
+        data = os.path.join(os.path.dirname(__file__), "data")
+        parser = get_parser()
+        parser.command = "nb2py"
+        parser.path = data
+        nb2py(data, verbose=1)
+        expected = os.path.join(data, "float_and_double_rouding.py")
+        self.assertExists(expected)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)
diff --git a/_unittests/ut__main/test_convert.py b/_unittests/ut__main/test_convert.py
@@ -0,0 +1,18 @@
+import platform
+import unittest
+import os
+from sphinx_runpython.ext_test_case import ExtTestCase
+from sphinx_runpython.convert import convert_ipynb_to_gallery
+
+
+class TestHelpers(ExtTestCase):
+    @unittest.skipIf(platform.system() != "Linux", reason="pandoc not installed")
+    def test_convert_notebook(self):
+        data = os.path.join(os.path.dirname(__file__), "data")
+        nb = os.path.join(data, "float_and_double_rouding.ipynb")
+        res = convert_ipynb_to_gallery(nb)
+        self.assertIn("# :math:`\\mathbb{P}", res)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -8,6 +8,7 @@ matplotlib
 numpydoc
 openpyxl
 pandas
+pypandoc
 pyquickhelper
 pytest
 pytest-cov
diff --git a/sphinx_runpython/__init__.py b/sphinx_runpython/__init__.py
@@ -1,7 +1,5 @@
-# coding: utf-8
-"""
-Sphinx extension.
-"""
-
 __version__ = "0.2.0"
 __author__ = "Xavier Dupré"
+__github__ = "https://github.com/sdpython/sphinx-runpython"
+__url__ = "https://sdpython.github.io/doc/sphinx-runpython/dev/"
+__license__ = "MIT License"
diff --git a/sphinx_runpython/__main__.py b/sphinx_runpython/__main__.py
@@ -0,0 +1,4 @@
+from ._cmd_helper import main
+
+if __name__ == "__main__":
+    main()
diff --git a/sphinx_runpython/_cmd_helper.py b/sphinx_runpython/_cmd_helper.py
@@ -0,0 +1,50 @@
+import glob
+import os
+from argparse import ArgumentParser
+from .convert import convert_ipynb_to_gallery
+
+
+def get_parser():
+    parser = ArgumentParser(
+        prog="sphinx-runpython command line",
+        description="A collection of quick tools.",
+        epilog="",
+    )
+    parser.add_argument("command", help="Command to run, only nb2py is available")
+    parser.add_argument(
+        "-p", "--path", help="Folder which contains the files to process"
+    )
+    parser.add_argument(
+        "-r", "--recursive", help="Recursive search.", action="store_true"
+    )
+    parser.add_argument("-v", "--verbose", help="verbosity", default=1, type=int)
+    return parser
+
+
+def nb2py(infolder: str, recursive: bool = False, verbose: int = 0):
+    if not os.path.exists(infolder):
+        raise FileNotFoundError(f"Unable to find {infolder!r}.")
+    patterns = [infolder + "/*.ipynb", infolder + "/**/*.ipynb"]
+    for pattern in patterns:
+        if verbose:
+            print(f"nb2py: look with pattern {pattern!r}, recursive={recursive}")
+        for name in glob.iglob(pattern, recursive=recursive):
+            spl = os.path.splitext(name)
+            out = spl[0] + ".py"
+            if verbose:
+                print(f"process {name!r} -> {out!r}")
+                convert_ipynb_to_gallery(name, outfile=out)
+
+
+def process_args(args):
+    cmd = args.command
+    if cmd == "nb2py":
+        nb2py(args.path, recursive=args.recursive, verbose=args.verbose)
+        return
+    raise ValueError(f"Command {cmd!r} is unknown.")
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+    process_args(args)
diff --git a/sphinx_runpython/convert.py b/sphinx_runpython/convert.py
@@ -0,0 +1,53 @@
+import json
+from typing import Optional
+import pypandoc as pdoc
+
+
+def convert_ipynb_to_gallery(nbfile: str, outfile: Optional[str] = None) -> str:
+    """
+    Convert jupyter notebook to sphinx gallery notebook styled examples.
+    It relies on :epkg:`pypandoc`.
+    Inspired from `ipynb_to_gallery.py
+    <https://gist.github.com/chsasank/7218ca16f8d022e02a9c0deb94a310fe>`_.
+
+    :param nbfile: notebook file
+    :param outfile: outpitfile
+    :return: content of this output file
+    """
+    python_file = []
+
+    with open(nbfile, "r", encoding="utf-8") as f:
+        nb_dict = json.load(f)
+    cells = nb_dict["cells"]
+
+    for i, cell in enumerate(cells):
+        if i == 0:
+            if cell["cell_type"] != "markdown":
+                raise ValueError(
+                    f"First cell has to be markdown but is {cell['cell_type']!r}."
+                )
+
+            md_source = "".join(cell["source"])
+            rst_source = pdoc.convert_text(md_source, "rst", "md")
+            python_file.append('"""\n' + rst_source + '\n"""')
+        else:
+            if cell["cell_type"] == "markdown":
+                md_source = "".join(cell["source"])
+                rst_source = pdoc.convert_text(md_source, "rst", "md")
+                commented_source = "\n".join(["# " + x for x in rst_source.split("\n")])
+                python_file.append("\n\n\n" + "#" * 70 + "\n" + commented_source)
+            elif cell["cell_type"] == "code":
+                source = "".join(cell["source"])
+                python_file.append("\n" * 2 + source)
+
+    python_file = "".join(python_file)
+
+    python_file = python_file.replace("\n%", "\n# %")
+    python_file = python_file.replace("’", "'")
+    python_file = python_file.replace("# %matplotlib inline", "")
+    python_file = python_file.replace("# \n", "\n")
+    python_file = python_file.replace(">`__", ">`_")
+    if outfile is None:
+        return python_file
+    with open(outfile, "w", encoding="utf-8") as f:
+        f.write(python_file)

-Original file line number
+Diff line change
     epkg
     gdot
     runpython
 +    tools
 Helpers
 =======