Skip to content

Commit aff037a

Browse files
partheaparthea
authored
chore: add scripts to update discovery artifacts (googleapis#1286)
These PR add the scripts from googleapis#1187 that are needed to update discovery artifacts using a Github action. The scripts will be removed from googleapis#1187 once all of the review comments from googleapis#1187 have been resolved. This PR adds the following files under the `scripts/` folder - `README.md` to provide instructions on manually updating discovery artifacts and API reference documentation. - `buildprbody.py` creates a summary of the changes detected in discovery artifacts and writes them to `allapis.summary`. - `changesummary.py` creates verbose change information for each API with discovery artifact changes. - `createcommits.sh` creates git commits for each API with discovery artifact changes or reference document changes. - `updatediscoveryartifacts.py` is the python file that can be used to update discovery artifacts. I also moved `describe.py` under the scripts folder and modified it to save the discovery artifacts that are fetched. TODO: - [x] Add tests for scripts - [x] Address review comments in googleapis#1187
1 parent ff1e936 commit aff037a

File tree

13 files changed

+19529
-72
lines changed

13 files changed

+19529
-72
lines changed

‎describe.py‎

Lines changed: 105 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import argparse
2929
import collections
3030
import json
31-
import os
31+
import pathlib
3232
import re
3333
import string
3434
import sys
@@ -37,12 +37,15 @@
3737
from googleapiclient.discovery import build
3838
from googleapiclient.discovery import build_from_document
3939
from googleapiclient.discovery import UnknownApiNameOrVersion
40-
from googleapiclient.discovery_cache import get_static_doc
4140
from googleapiclient.http import build_http
4241
from googleapiclient.errors import HttpError
4342

4443
import uritemplate
4544

45+
DISCOVERY_DOC_DIR = (
46+
pathlib.Path(__file__).parent.resolve() / "googleapiclient" / "discovery_cache" / "documents"
47+
)
48+
4649
CSS = """<style>
4750
4851
body, h1, h2, h3, div, span, p, pre, a {
@@ -133,7 +136,7 @@
133136
<code><a href="#$name">$name($params)</a></code></p>
134137
<p class="firstline">$firstline</p>"""
135138

136-
BASE = "docs/dyn"
139+
BASE = pathlib.Path(__file__).parent.resolve() / "docs" / "dyn"
137140

138141
DIRECTORY_URI = "https://www.googleapis.com/discovery/v1/apis"
139142

@@ -254,14 +257,10 @@ def method(name, doc):
254257
name: string, Name of the method.
255258
doc: string, The methods docstring.
256259
"""
260+
import html
257261

258262
params = method_params(doc)
259-
if sys.version_info.major >= 3:
260-
import html
261-
doc = html.escape(doc)
262-
else:
263-
import cgi
264-
doc = cgi.escape(doc)
263+
doc = html.escape(doc)
265264
return string.Template(METHOD_TEMPLATE).substitute(
266265
name=name, params=params, doc=doc
267266
)
@@ -358,13 +357,10 @@ def document_collection(resource, path, root_discovery, discovery, css=CSS):
358357
return "\n".join(html)
359358

360359

361-
def document_collection_recursive(resource, path, root_discovery, discovery):
362-
360+
def document_collection_recursive(resource, path, root_discovery, discovery, doc_destination_dir):
363361
html = document_collection(resource, path, root_discovery, discovery)
364362

365-
f = open(os.path.join(FLAGS.dest, path + "html"), "w")
366-
if sys.version_info.major < 3:
367-
html = html.encode("utf-8")
363+
f = open(pathlib.Path(doc_destination_dir).joinpath(path + "html"), "w")
368364

369365
f.write(html)
370366
f.close()
@@ -383,44 +379,76 @@ def document_collection_recursive(resource, path, root_discovery, discovery):
383379
path + name + ".",
384380
root_discovery,
385381
discovery["resources"].get(dname, {}),
382+
doc_destination_dir
386383
)
387384

388385

389-
def document_api(name, version, uri):
386+
def document_api(name, version, uri, doc_destination_dir):
390387
"""Document the given API.
391388
392-
Args:
393-
name: string, Name of the API.
394-
version: string, Version of the API.
395-
uri: string, URI of the API's discovery document
389+
Args:
390+
name (str): Name of the API.
391+
version (str): Version of the API.
392+
uri (str): URI of the API's discovery document
393+
doc_destination_dir (str): relative path where the reference
394+
documentation should be saved.
396395
"""
397-
try:
398-
service = build(name, version)
399-
content = get_static_doc(name, version)
400-
except UnknownApiNameOrVersion as e:
401-
print("Warning: {} {} found but could not be built.".format(name, version))
396+
http = build_http()
397+
resp, content = http.request(
398+
uri or uritemplate.expand(
399+
FLAGS.discovery_uri_template, {"api": name, "apiVersion": version}
400+
)
401+
)
402+
403+
if resp.status == 200:
404+
discovery = json.loads(content)
405+
service = build_from_document(discovery)
406+
version = safe_version(version)
407+
doc_name = "{}.{}.json".format(name, version.replace("_", ""))
408+
409+
discovery_file_path = DISCOVERY_DOC_DIR / doc_name
410+
revision = None
411+
412+
pathlib.Path(discovery_file_path).touch(exist_ok=True)
413+
414+
# Write discovery artifact to disk if revision equal or newer
415+
with open(discovery_file_path, "r+") as f:
416+
try:
417+
json_data = json.load(f)
418+
revision = json_data['revision']
419+
except json.JSONDecodeError:
420+
revision = None
421+
422+
if revision is None or discovery['revision'] >= revision:
423+
# Reset position to the beginning
424+
f.seek(0)
425+
# Write the changes to disk
426+
json.dump(discovery, f, indent=2, sort_keys=True)
427+
# Truncate anything left as it's not needed
428+
f.truncate()
429+
430+
elif resp.status == 404:
431+
print("Warning: {} {} not found. HTTP Code: {}".format(name, version, resp.status))
402432
return
403-
except HttpError as e:
404-
print("Warning: {} {} returned {}.".format(name, version, e))
433+
else:
434+
print("Warning: {} {} could not be built. HTTP Code: {}".format(name, version, resp.status))
405435
return
406436

407-
discovery = json.loads(content)
408-
409-
version = safe_version(version)
410-
411437
document_collection_recursive(
412-
service, "{}_{}.".format(name, version), discovery, discovery
438+
service, "{}_{}.".format(name, version), discovery, discovery, doc_destination_dir
413439
)
414440

415441

416-
def document_api_from_discovery_document(uri):
442+
def document_api_from_discovery_document(discovery_url, doc_destination_dir):
417443
"""Document the given API.
418444
419445
Args:
420-
uri: string, URI of discovery document.
446+
discovery_url (str): URI of discovery document.
447+
doc_destination_dir (str): relative path where the reference
448+
documentation should be saved.
421449
"""
422450
http = build_http()
423-
response, content = http.request(FLAGS.discovery_uri)
451+
response, content = http.request(discovery_url)
424452
discovery = json.loads(content)
425453

426454
service = build_from_document(discovery)
@@ -429,48 +457,53 @@ def document_api_from_discovery_document(uri):
429457
version = safe_version(discovery["version"])
430458

431459
document_collection_recursive(
432-
service, "{}_{}.".format(name, version), discovery, discovery
460+
service, "{}_{}.".format(name, version), discovery, discovery, doc_destination_dir
433461
)
434462

463+
def generate_all_api_documents(directory_uri=DIRECTORY_URI, doc_destination_dir=BASE):
464+
""" Retrieve discovery artifacts and fetch reference documentations
465+
for all apis listed in the public discovery directory.
466+
args:
467+
directory_uri (str): uri of the public discovery directory.
468+
doc_destination_dir (str): relative path where the reference
469+
documentation should be saved.
470+
"""
471+
api_directory = collections.defaultdict(list)
472+
http = build_http()
473+
resp, content = http.request(directory_uri)
474+
if resp.status == 200:
475+
directory = json.loads(content)["items"]
476+
for api in directory:
477+
document_api(api["name"], api["version"], api["discoveryRestUrl"], doc_destination_dir)
478+
api_directory[api["name"]].append(api["version"])
479+
480+
# sort by api name and version number
481+
for api in api_directory:
482+
api_directory[api] = sorted(api_directory[api])
483+
api_directory = OrderedDict(
484+
sorted(api_directory.items(), key=lambda x: x[0])
485+
)
486+
487+
markdown = []
488+
for api, versions in api_directory.items():
489+
markdown.append("## %s" % api)
490+
for version in versions:
491+
markdown.append(
492+
"* [%s](http://googleapis.github.io/google-api-python-client/docs/dyn/%s_%s.html)"
493+
% (version, api, safe_version(version))
494+
)
495+
markdown.append("\n")
496+
497+
with open(BASE / "index.md", "w") as f:
498+
markdown = "\n".join(markdown)
499+
f.write(markdown)
500+
501+
else:
502+
sys.exit("Failed to load the discovery document.")
435503

436504
if __name__ == "__main__":
437505
FLAGS = parser.parse_args(sys.argv[1:])
438506
if FLAGS.discovery_uri:
439-
document_api_from_discovery_document(FLAGS.discovery_uri)
507+
document_api_from_discovery_document(discovery_url=FLAGS.discovery_uri, doc_destination_dir=FLAGS.dest)
440508
else:
441-
api_directory = collections.defaultdict(list)
442-
http = build_http()
443-
resp, content = http.request(
444-
FLAGS.directory_uri, headers={"X-User-IP": "0.0.0.0"}
445-
)
446-
if resp.status == 200:
447-
directory = json.loads(content)["items"]
448-
for api in directory:
449-
document_api(api["name"], api["version"], api["discoveryRestUrl"])
450-
api_directory[api["name"]].append(api["version"])
451-
452-
# sort by api name and version number
453-
for api in api_directory:
454-
api_directory[api] = sorted(api_directory[api])
455-
api_directory = OrderedDict(
456-
sorted(api_directory.items(), key=lambda x: x[0])
457-
)
458-
459-
markdown = []
460-
for api, versions in api_directory.items():
461-
markdown.append("## %s" % api)
462-
for version in versions:
463-
markdown.append(
464-
"* [%s](http://googleapis.github.io/google-api-python-client/docs/dyn/%s_%s.html)"
465-
% (version, api, safe_version(version))
466-
)
467-
markdown.append("\n")
468-
469-
with open("docs/dyn/index.md", "w") as f:
470-
markdown = "\n".join(markdown)
471-
if sys.version_info.major < 3:
472-
markdown = markdown.encode("utf-8")
473-
f.write(markdown)
474-
475-
else:
476-
sys.exit("Failed to load the discovery document.")
509+
generate_all_api_documents(directory_uri=FLAGS.directory_uri, doc_destination_dir=FLAGS.dest)

‎noxfile.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16+
import os
1617
import sys
1718

1819
import nox
@@ -90,3 +91,21 @@ def unit(session, oauth2client):
9091
"tests",
9192
*session.posargs,
9293
)
94+
95+
@nox.session(python=["3.9"])
96+
def scripts(session):
97+
session.install(*test_dependencies)
98+
session.install("-e", ".")
99+
session.install("-r", "scripts/requirements.txt")
100+
101+
# Run py.test against the unit tests.
102+
session.run(
103+
"py.test",
104+
"--quiet",
105+
"--cov=scripts",
106+
"--cov-config=.coveragerc",
107+
"--cov-report=",
108+
"--cov-fail-under=80",
109+
"scripts",
110+
*session.posargs,
111+
)

‎scripts/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Discovery Artifact Automation
2+
Discovery Artifacts are automatically updated using a Github Action script. This
3+
documentation is intended for users that need to maintain the repository.
4+
5+
## Updating discovery artifacts locally
6+
7+
To update discovery artifacts locally:
8+
1. Create a virtual environment using `pyenv virtualenv updateartifacts`
9+
2. Activate the virtual environment using `pyenv activate updateartifacts`
10+
3. Clone the repository, and `cd` into the `scripts` directory
11+
4. Run `pip install -r requirements.txt`
12+
5. Run `pip install -e ../`
13+
6. Run `git checkout -b update-discovery-artifacts-manual`
14+
7. Run `python3 updatediscoveryartifacts.py`
15+
8. Run `./createcommits.sh`
16+
9. Run `python3 buildprbody.py`
17+
10. Create a pull request with the changes.
18+
11. Copy the contents of `temp/allapis.summary` into the PR Body.
19+
20+
## Questions
21+
Feel free to submit an issue!

0 commit comments

Comments
 (0)