Refactoring from feedback. Add tox support

daspecster · daspecster · commit 4db872325038 · 2016-05-05T18:06:51.000-04:00
diff --git a/docs/json/mastertoc.json b/docs/json/mastertoc.json
diff --git a/scripts/generate_json_docs.py b/scripts/generate_json_docs.py
@@ -19,10 +19,11 @@
 import os
 import sys
 import types
-
 from shutil import copyfile
+
 from parinx.parser import parse_docstring
 import pdoc
+
 from verify_included_modules import get_public_modules
 
 
@@ -41,9 +42,13 @@ def from_module_name(cls, name):
         for c in module.classes():
             methods.extend(c.doc.values())
 
+        mod = __import__(name)
+        examples = json.dumps(get_module_examples(mod))
+
         return cls(module_id=name,
                    name=name.split('.')[-1].title(),
                    description=module.docstring,
+                   examples=examples,
                    methods=[Method.from_pdoc(m) for m in methods])
 
     def to_dict(self):
@@ -103,17 +108,19 @@ def from_pdoc(cls, element):
 
             line = inspect.getsourcelines(mod)[1]
             source_path = inspect.getsourcefile(mod)
+
+            # Cleanup path so it routes for site builder.
             for path in sys.path:
                 source_path = source_path.replace(path, '')
+            source_path = source_path.replace('/site-packages/gcloud', '')
 
-            source_path = source_path + "#L" + str(line)
+            source_path = source_path + '#L' + str(line)
             method.add_source_line(source_path)
 
-            # Sketchy get examples from method docstring.
-            mod_doctest = doctest.DocTestFinder().find(mod)
-            for md in mod_doctest:
-                for example in md.examples:
-                    method.add_example(example.source)
+            # Sketchy get examples from method docstring.\
+            examples = get_module_examples(mod)
+            for example in examples:
+                method.add_example(example)
 
         if element.docstring:
             if not isinstance(element, pdoc.Class) and element.cls:
@@ -125,9 +132,14 @@ def from_pdoc(cls, element):
 
             # Hack for old-style classes
             if str(cls)[0] != '<':
-                cls = "<class '" + str(cls) + "'>"
+                cls = '<class \'' + str(cls) + '\'>'
 
-            method_info = parse_docstring(element.docstring, cls)
+            try:
+                method_info = parse_docstring(element.docstring, cls)
+            except Exception as err:
+                print err.message
+                print method.name
+                return method
 
             for name, data in method_info['arguments'].items():
                 param = Param.from_docstring_section(name, data)
@@ -166,6 +178,16 @@ def from_docstring_section(cls, name, data):
                    types=[data['type_name']])
 
 
+def get_module_examples(module):
+    examples = []
+    module_doctest = doctest.DocTestFinder().find(module)
+    for doctest_obj in module_doctest:
+        for example in doctest_obj.examples:
+            example_str = "%s\n%s" % (example.source, example.want)
+            examples.append(example_str)
+    return examples
+
+
 def write_docs_file(path, contents):
     if not os.path.exists(os.path.dirname(path)):
         try:
@@ -178,17 +200,17 @@ def write_docs_file(path, contents):
 
 def generate_doc_types_json(modules, types_file_path):
     doc_types_list = [{
-        "id": "gcloud",
-        "contents": "__init__.json",
-        "title": "__Init__"
+        'id': 'gcloud',
+        'contents': '__init__.json',
+        'title': '__Init__'
     }]
 
     for module_name in modules:
         module_path = module_name.replace('.', '/').replace('gcloud/', '')
         doc_type_object = {
-            "id": module_path,
-            "title": u' \xbb '.join(module_path.split('/')).title(),
-            "contents": module_path + '.json'
+            'id': module_path,
+            'title': u' \xbb '.join(module_path.split('/')).title(),
+            'contents': module_path + '.json'
         }
         doc_types_list.append(doc_type_object)
 
@@ -200,29 +222,27 @@ def generate_module_docs(modules, base_path):
     for module_name in modules:
         module = Module.from_module_name(module_name)
         module_path = module_name.replace('.', '/').replace('gcloud/', '')
-        module_docs_path = (base_path +
-                            '/' + module_path + '.json')
+        module_docs_path = os.path.join(base_path, module_path) + '.json'
 
         write_docs_file(module_docs_path, json.dumps(module.to_dict(),
                                                      indent=2, sort_keys=True))
 
 
 def main():
-    BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-    DOCS_DIR = os.path.join(BASE_DIR, 'docs') + '/json/'
-
     parser = argparse.ArgumentParser(description='Document Python modules.')
     parser.add_argument('tag', help='The version of the documentation.')
     args = parser.parse_args()
 
+    BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+    DOCS_DIR = os.path.join(BASE_DIR, 'docs', 'json', args.tag)
+
     library_dir = os.path.join(BASE_DIR, 'gcloud')
-    public_mods = get_public_modules(library_dir,
-                                     base_package='gcloud')
+    public_mods = get_public_modules(library_dir, base_package='gcloud')
 
-    generate_module_docs(public_mods, DOCS_DIR + args.tag)
-    generate_doc_types_json(public_mods, DOCS_DIR + args.tag + '/types.json')
+    generate_module_docs(public_mods, DOCS_DIR)
+    generate_doc_types_json(public_mods, DOCS_DIR + '/types.json')
 
-    copyfile(DOCS_DIR + 'toc.json', DOCS_DIR + args.tag + '/toc.json')
+    copyfile(DOCS_DIR + 'toc.json', DOCS_DIR + '/toc.json')
 
 if __name__ == '__main__':
     main()
diff --git a/tox.ini b/tox.ini
@@ -73,6 +73,18 @@ deps =
     sphinx_rtd_theme
 passenv = {[testenv:system-tests]passenv} SPHINX_RELEASE READTHEDOCS LOCAL_RTD
 
+[testenv:json-docs]
+basepython =
+    python2.7
+commands =
+    python scripts/generate_json_docs.py master
+deps =
+    parinx
+    pdoc
+    Sphinx
+setenv =
+    PYTHONPATH = {toxinidir}/_testing
+
 [testenv:docs-rtd]
 setenv =
     PYTHONPATH = {toxinidir}/_testing
