Skip to content

Commit bdf7cc2

Browse files
coryancoryan
authored
ci: use Python venv for docuploader (#11997)
`docuploader` is the tool used to upload our reference douments to Google. With this change the tool is installed in an isolated Python virtual environment, so its dependencies do not conflict with the deps for other tools (e.g. the storage testbench).
1 parent 35f552e commit bdf7cc2

3 files changed

Lines changed: 119 additions & 38 deletions

File tree

‎ci/cloudbuild/builds/publish-docs.sh‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -83,25 +83,19 @@ fi
8383
# `pip install` commands. Googlers may want to see b/242562806 for more
8484
# details.
8585
#
86-
# Because we are currently using a SHA (as opposed to a released version) of
87-
# the `gcp-docuploader` the process to update these hashes is somewhat manual.
88-
# This worked for me (coryan@) circa 2022-09-08:
89-
#
9086
# venv="$(mktemp -d)"
9187
# python3 -m venv "${venv}/requirements"
9288
# source "${venv}/requirements/bin/activate"
93-
# pip install git+https://github.com/googleapis/docuploader@993badb47be3bf548a4c1726658eadba4bafeaca
94-
# pip freeze | grep -v gcp-docuploader >ci/etc/docuploader-requirements.in
89+
# pip install gcp-docuploader
90+
# pip freeze >ci/etc/docuploader-requirements.in
9591
# pip install --require-hashes -r ci/etc/pip-tooling-requirements.txt
9692
# pip-compile --generate-hashes ci/etc/docuploader-requirements.in
9793
#
9894
io::log_h2 "Installing the docuploader package"
99-
python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
95+
python3 -m venv "${HOME}/.venv/docuploader"
96+
source "${HOME}/.venv/docuploader/bin/activate"
97+
python3 -m pip install --upgrade --quiet --disable-pip-version-check \
10098
--no-warn-script-location --require-hashes -r ci/etc/docuploader-requirements.txt
101-
python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
102-
--no-warn-script-location --no-deps \
103-
"git+https://github.com/googleapis/docuploader@993badb47be3bf548a4c1726658eadba4bafeaca"
104-
python3 -m pip list
10599

106100
# For docuploader to work
107101
export LC_ALL=C.UTF-8

‎ci/etc/docuploader-requirements.in‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
1-
cachetools==5.2.0
2-
certifi==2022.6.15
3-
charset-normalizer==2.1.1
1+
cachetools==5.3.1
2+
certifi==2023.5.7
3+
charset-normalizer==3.1.0
44
click==8.1.3
55
colorlog==6.7.0
6-
google-api-core==2.10.0
7-
google-auth==2.11.0
6+
gcp-docuploader==0.6.5
7+
google-api-core==2.11.1
8+
google-auth==2.21.0
89
google-cloud-core==2.3.2
9-
google-cloud-storage==2.5.0
10+
google-cloud-storage==2.10.0
1011
google-crc32c==1.5.0
11-
google-resumable-media==2.3.3
12-
googleapis-common-protos==1.56.4
13-
idna==3.3
14-
protobuf==4.21.5
15-
pyasn1==0.4.8
16-
pyasn1-modules==0.2.8
17-
requests==2.28.1
12+
google-resumable-media==2.5.0
13+
googleapis-common-protos==1.59.1
14+
idna==3.4
15+
protobuf==4.23.3
16+
pyasn1==0.5.0
17+
pyasn1-modules==0.3.0
18+
requests==2.31.0
1819
rsa==4.9
1920
six==1.16.0
20-
urllib3==1.26.12
21+
urllib3==1.26.16

‎ci/etc/docuploader-requirements.txt‎

Lines changed: 99 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#
2-
# This file is autogenerated by pip-compile with python 3.10
3-
# To update, run:
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
44
#
55
# pip-compile --generate-hashes ci/etc/docuploader-requirements.in
66
#
@@ -16,19 +16,100 @@ certifi==2023.5.7 \
1616
# via
1717
# -r ci/etc/docuploader-requirements.in
1818
# requests
19-
charset-normalizer==2.1.1 \
20-
--hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \
21-
--hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f
19+
charset-normalizer==3.1.0 \
20+
--hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \
21+
--hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \
22+
--hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \
23+
--hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \
24+
--hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \
25+
--hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \
26+
--hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \
27+
--hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \
28+
--hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \
29+
--hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \
30+
--hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \
31+
--hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \
32+
--hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \
33+
--hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \
34+
--hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \
35+
--hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \
36+
--hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \
37+
--hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \
38+
--hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \
39+
--hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \
40+
--hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \
41+
--hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \
42+
--hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \
43+
--hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \
44+
--hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \
45+
--hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \
46+
--hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \
47+
--hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \
48+
--hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \
49+
--hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \
50+
--hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \
51+
--hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \
52+
--hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \
53+
--hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \
54+
--hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \
55+
--hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \
56+
--hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \
57+
--hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \
58+
--hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \
59+
--hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \
60+
--hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \
61+
--hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \
62+
--hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \
63+
--hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \
64+
--hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \
65+
--hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \
66+
--hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \
67+
--hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \
68+
--hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \
69+
--hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \
70+
--hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \
71+
--hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \
72+
--hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \
73+
--hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \
74+
--hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \
75+
--hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \
76+
--hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \
77+
--hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \
78+
--hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \
79+
--hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \
80+
--hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \
81+
--hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \
82+
--hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \
83+
--hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \
84+
--hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \
85+
--hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \
86+
--hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \
87+
--hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \
88+
--hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \
89+
--hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \
90+
--hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \
91+
--hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \
92+
--hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \
93+
--hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \
94+
--hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab
2295
# via
2396
# -r ci/etc/docuploader-requirements.in
2497
# requests
2598
click==8.1.3 \
2699
--hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \
27100
--hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
28-
# via -r ci/etc/docuploader-requirements.in
101+
# via
102+
# -r ci/etc/docuploader-requirements.in
103+
# gcp-docuploader
29104
colorlog==6.7.0 \
30105
--hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \
31106
--hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5
107+
# via
108+
# -r ci/etc/docuploader-requirements.in
109+
# gcp-docuploader
110+
gcp-docuploader==0.6.5 \
111+
--hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \
112+
--hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea
32113
# via -r ci/etc/docuploader-requirements.in
33114
google-api-core==2.11.1 \
34115
--hash=sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a \
@@ -37,9 +118,9 @@ google-api-core==2.11.1 \
37118
# -r ci/etc/docuploader-requirements.in
38119
# google-cloud-core
39120
# google-cloud-storage
40-
google-auth==2.17.3 \
41-
--hash=sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc \
42-
--hash=sha256:f586b274d3eb7bd932ea424b1c702a30e0393a2e2bc4ca3eae8263ffd8be229f
121+
google-auth==2.21.0 \
122+
--hash=sha256:b28e8048e57727e7cf0e5bd8e7276b212aef476654a09511354aa82753b45c66 \
123+
--hash=sha256:da3f18d074fa0f5a7061d99b9af8cee3aa6189c987af7c1b07d94566b6b11268
43124
# via
44125
# -r ci/etc/docuploader-requirements.in
45126
# google-api-core
@@ -54,7 +135,9 @@ google-cloud-core==2.3.2 \
54135
google-cloud-storage==2.10.0 \
55136
--hash=sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7 \
56137
--hash=sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7
57-
# via -r ci/etc/docuploader-requirements.in
138+
# via
139+
# -r ci/etc/docuploader-requirements.in
140+
# gcp-docuploader
58141
google-crc32c==1.5.0 \
59142
--hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \
60143
--hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \
@@ -161,6 +244,7 @@ protobuf==4.23.3 \
161244
--hash=sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5
162245
# via
163246
# -r ci/etc/docuploader-requirements.in
247+
# gcp-docuploader
164248
# google-api-core
165249
# googleapis-common-protos
166250
pyasn1==0.5.0 \
@@ -194,10 +278,12 @@ six==1.16.0 \
194278
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
195279
# via
196280
# -r ci/etc/docuploader-requirements.in
281+
# gcp-docuploader
197282
# google-auth
198-
urllib3==2.0.3 \
199-
--hash=sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1 \
200-
--hash=sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825
283+
urllib3==1.26.16 \
284+
--hash=sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f \
285+
--hash=sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14
201286
# via
202287
# -r ci/etc/docuploader-requirements.in
288+
# google-auth
203289
# requests

0 commit comments

Comments
 (0)