Skip to content

Commit 7d1161f

Browse files
author
Jon Wayne Parrott
authored
Remove httplib2, replace with Requests (googleapis#3674)
* Core: remove httplib2, replace with Requests Additionally remove make_exception in favor of from_http_status and from_http_response. * Datastore: replace httplib2 with Requests * DNS: replace httplib2 with Requests * Error Reporting: replace httplib2 with requests * Language: replace httplib2 with Requests * Logging: replace httplib2 with requests * Monitoring: replace httplib2 with Requests * Pubsub: replace httplib2 with Requests * Resource Manager: replace httplib2 with Requests * Runtimeconfig: replace httplib2 with Requests * Speech: replace httplib2 with Requests * Storage: replace httplib2 with Requests * BigQuery: replace httplib2 with Requests * Translate: replace httplib2 with Requests * Vision: replace httplib2 with Requests
1 parent 6fff098 commit 7d1161f

File tree

56 files changed

+1050
-1201
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1050
-1201
lines changed

‎bigquery/google/cloud/bigquery/client.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ class Client(ClientWithProject):
6464
passed), falls back to the default inferred from the
6565
environment.
6666
67-
:type _http: :class:`~httplib2.Http`
67+
:type _http: :class:`~requests.Session`
6868
:param _http: (Optional) HTTP object to make requests. Can be any object
6969
that defines ``request()`` with the same interface as
70-
:meth:`~httplib2.Http.request`. If not passed, an
70+
:meth:`requests.Session.request`. If not passed, an
7171
``_http`` object is created that is bound to the
7272
``credentials`` for the current object.
7373
This parameter should be considered private, and could

‎bigquery/google/cloud/bigquery/dataset.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ def _parse_access_grants(access):
364364
def _set_properties(self, api_response):
365365
"""Update properties from resource in body of ``api_response``
366366
367-
:type api_response: httplib2.Response
367+
:type api_response: dict
368368
:param api_response: response returned from an API call.
369369
"""
370370
self._properties.clear()

‎bigquery/google/cloud/bigquery/job.py‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
"""Define API Jobs."""
1616

17-
import collections
1817
import threading
1918

2019
import six
@@ -58,8 +57,6 @@
5857
'tableUnavailable': http_client.BAD_REQUEST,
5958
}
6059

61-
_FakeResponse = collections.namedtuple('_FakeResponse', ['status'])
62-
6360

6461
def _error_result_to_exception(error_result):
6562
"""Maps BigQuery error reasons to an exception.
@@ -79,13 +76,8 @@ def _error_result_to_exception(error_result):
7976
reason = error_result.get('reason')
8077
status_code = _ERROR_REASON_TO_EXCEPTION.get(
8178
reason, http_client.INTERNAL_SERVER_ERROR)
82-
# make_exception expects an httplib2 response object.
83-
fake_response = _FakeResponse(status=status_code)
84-
return exceptions.make_exception(
85-
fake_response,
86-
error_result.get('message', ''),
87-
error_info=error_result,
88-
use_json=False)
79+
return exceptions.from_http_status(
80+
status_code, error_result.get('message', ''), errors=[error_result])
8981

9082

9183
class Compression(_EnumProperty):
@@ -307,7 +299,7 @@ def _scrub_local_properties(self, cleaned):
307299
def _set_properties(self, api_response):
308300
"""Update properties from resource in body of ``api_response``
309301
310-
:type api_response: httplib2.Response
302+
:type api_response: dict
311303
:param api_response: response returned from an API call
312304
"""
313305
cleaned = api_response.copy()

‎bigquery/google/cloud/bigquery/query.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ def schema(self):
310310
def _set_properties(self, api_response):
311311
"""Update properties from resource in body of ``api_response``
312312
313-
:type api_response: httplib2.Response
313+
:type api_response: dict
314314
:param api_response: response returned from an API call
315315
"""
316316
self._properties.clear()

‎bigquery/google/cloud/bigquery/table.py‎

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,16 @@
1717
import datetime
1818
import os
1919

20-
import httplib2
2120
import six
2221

2322
import google.auth.transport.requests
2423
from google import resumable_media
2524
from google.resumable_media.requests import MultipartUpload
2625
from google.resumable_media.requests import ResumableUpload
2726

27+
from google.cloud import exceptions
2828
from google.cloud._helpers import _datetime_from_microseconds
2929
from google.cloud._helpers import _millis_from_datetime
30-
from google.cloud.exceptions import NotFound
31-
from google.cloud.exceptions import make_exception
3230
from google.cloud.iterator import HTTPIterator
3331
from google.cloud.bigquery.schema import SchemaField
3432
from google.cloud.bigquery._helpers import _item_to_row
@@ -474,7 +472,7 @@ def _require_client(self, client):
474472
def _set_properties(self, api_response):
475473
"""Update properties from resource in body of ``api_response``
476474
477-
:type api_response: httplib2.Response
475+
:type api_response: dict
478476
:param api_response: response returned from an API call
479477
"""
480478
self._properties.clear()
@@ -563,7 +561,7 @@ def exists(self, client=None):
563561
try:
564562
client._connection.api_request(method='GET', path=self.path,
565563
query_params={'fields': 'id'})
566-
except NotFound:
564+
except exceptions.NotFound:
567565
return False
568566
else:
569567
return True
@@ -1113,7 +1111,7 @@ def upload_from_file(self,
11131111
client, file_obj, metadata, size, num_retries)
11141112
return client.job_from_resource(created_json)
11151113
except resumable_media.InvalidResponse as exc:
1116-
_raise_from_invalid_response(exc)
1114+
raise exceptions.from_http_response(exc.response)
11171115
# pylint: enable=too-many-arguments,too-many-locals
11181116

11191117

@@ -1298,22 +1296,3 @@ def _get_upload_metadata(source_format, schema, dataset, name):
12981296
'load': load_config,
12991297
},
13001298
}
1301-
1302-
1303-
def _raise_from_invalid_response(error, error_info=None):
1304-
"""Re-wrap and raise an ``InvalidResponse`` exception.
1305-
1306-
:type error: :exc:`google.resumable_media.InvalidResponse`
1307-
:param error: A caught exception from the ``google-resumable-media``
1308-
library.
1309-
1310-
:type error_info: str
1311-
:param error_info: (Optional) Extra information about the failed request.
1312-
1313-
:raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
1314-
to the failed status code
1315-
"""
1316-
response = error.response
1317-
faux_response = httplib2.Response({'status': response.status_code})
1318-
raise make_exception(faux_response, response.content,
1319-
error_info=error_info, use_json=False)

‎bigquery/tests/unit/test__http.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import unittest
1616

1717
import mock
18+
import requests
1819

1920

2021
class TestConnection(unittest.TestCase):
@@ -55,10 +56,12 @@ def test_extra_headers(self):
5556
from google.cloud import _http as base_http
5657
from google.cloud.bigquery import _http as MUT
5758

58-
http = mock.Mock(spec=['request'])
59-
response = mock.Mock(status=200, spec=['status'])
59+
http = mock.create_autospec(requests.Session, instance=True)
60+
response = requests.Response()
61+
response.status_code = 200
6062
data = b'brent-spiner'
61-
http.request.return_value = response, data
63+
response._content = data
64+
http.request.return_value = response
6265
client = mock.Mock(_http=http, spec=['_http'])
6366

6467
conn = self._make_one(client)
@@ -68,15 +71,14 @@ def test_extra_headers(self):
6871
self.assertEqual(result, data)
6972

7073
expected_headers = {
71-
'Content-Length': str(len(req_data)),
7274
'Accept-Encoding': 'gzip',
7375
base_http.CLIENT_INFO_HEADER: MUT._CLIENT_INFO,
7476
'User-Agent': conn.USER_AGENT,
7577
}
7678
expected_uri = conn.build_api_url('/rainbow')
7779
http.request.assert_called_once_with(
78-
body=req_data,
80+
data=req_data,
7981
headers=expected_headers,
8082
method='GET',
81-
uri=expected_uri,
83+
url=expected_uri,
8284
)

‎bigquery/tests/unit/test_job.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def test_simple(self):
3131
exception = self._call_fut(error_result)
3232
self.assertEqual(exception.code, http_client.BAD_REQUEST)
3333
self.assertTrue(exception.message.startswith('bad request'))
34-
self.assertIn("'reason': 'invalid'", exception.message)
34+
self.assertIn(error_result, exception.errors)
3535

3636
def test_missing_reason(self):
3737
error_result = {}

‎bigquery/tests/unit/test_table.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,9 +1744,11 @@ def _make_table():
17441744
def _make_response(status_code, content='', headers={}):
17451745
"""Make a mock HTTP response."""
17461746
import requests
1747-
response = mock.create_autospec(requests.Response, instance=True)
1748-
response.content = content.encode('utf-8')
1749-
response.headers = headers
1747+
response = requests.Response()
1748+
response.request = requests.Request(
1749+
'POST', 'http://example.com').prepare()
1750+
response._content = content.encode('utf-8')
1751+
response.headers.update(headers)
17501752
response.status_code = status_code
17511753
return response
17521754

@@ -1921,7 +1923,7 @@ def test_upload_from_file_failure(self):
19211923
table.upload_from_file(
19221924
file_obj, source_format='CSV', rewind=True)
19231925

1924-
assert exc_info.value.message == response.content.decode('utf-8')
1926+
assert response.text in exc_info.value.message
19251927
assert exc_info.value.errors == []
19261928

19271929
def test_upload_from_file_bad_mode(self):

‎core/google/cloud/_helpers.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,11 @@
2525
import re
2626
from threading import local as Local
2727

28-
import google_auth_httplib2
29-
import httplib2
3028
import six
3129
from six.moves import http_client
3230

3331
import google.auth
32+
import google.auth.transport.requests
3433
from google.protobuf import duration_pb2
3534
from google.protobuf import timestamp_pb2
3635

@@ -550,7 +549,7 @@ def make_secure_channel(credentials, user_agent, host, extra_options=()):
550549
:returns: gRPC secure channel with credentials attached.
551550
"""
552551
target = '%s:%d' % (host, http_client.HTTPS_PORT)
553-
http_request = google_auth_httplib2.Request(http=httplib2.Http())
552+
http_request = google.auth.transport.requests.Request()
554553

555554
user_agent_option = ('grpc.primary_user_agent', user_agent)
556555
options = (user_agent_option,) + extra_options

‎core/google/cloud/_http.py‎

Lines changed: 18 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@
1818
import platform
1919

2020
from pkg_resources import get_distribution
21-
import six
2221
from six.moves.urllib.parse import urlencode
2322

24-
from google.cloud.exceptions import make_exception
23+
from google.cloud import exceptions
2524

2625

2726
API_BASE_URL = 'https://www.googleapis.com'
@@ -67,8 +66,9 @@ def credentials(self):
6766
def http(self):
6867
"""A getter for the HTTP transport used in talking to the API.
6968
70-
:rtype: :class:`httplib2.Http`
71-
:returns: A Http object used to transport data.
69+
Returns:
70+
google.auth.transport.requests.AuthorizedSession:
71+
A :class:`requests.Session` instance.
7272
"""
7373
return self._client._http
7474

@@ -168,23 +168,13 @@ def _make_request(self, method, url, data=None, content_type=None,
168168
custom behavior, for example, to defer an HTTP request and complete
169169
initialization of the object at a later time.
170170
171-
:rtype: tuple of ``response`` (a dictionary of sorts)
172-
and ``content`` (a string).
173-
:returns: The HTTP response object and the content of the response,
174-
returned by :meth:`_do_request`.
171+
:rtype: :class:`requests.Response`
172+
:returns: The HTTP response.
175173
"""
176174
headers = headers or {}
177175
headers.update(self._EXTRA_HEADERS)
178176
headers['Accept-Encoding'] = 'gzip'
179177

180-
if data:
181-
content_length = len(str(data))
182-
else:
183-
content_length = 0
184-
185-
# NOTE: str is intended, bytes are sufficient for headers.
186-
headers['Content-Length'] = str(content_length)
187-
188178
if content_type:
189179
headers['Content-Type'] = content_type
190180

@@ -215,12 +205,11 @@ def _do_request(self, method, url, headers, data,
215205
(Optional) Unused ``target_object`` here but may be used by a
216206
superclass.
217207
218-
:rtype: tuple of ``response`` (a dictionary of sorts)
219-
and ``content`` (a string).
220-
:returns: The HTTP response object and the content of the response.
208+
:rtype: :class:`requests.Response`
209+
:returns: The HTTP response.
221210
"""
222-
return self.http.request(uri=url, method=method, headers=headers,
223-
body=data)
211+
return self.http.request(
212+
url=url, method=method, headers=headers, data=data)
224213

225214
def api_request(self, method, path, query_params=None,
226215
data=None, content_type=None, headers=None,
@@ -281,7 +270,7 @@ def api_request(self, method, path, query_params=None,
281270
282271
:raises ~google.cloud.exceptions.GoogleCloudError: if the response code
283272
is not 200 OK.
284-
:raises TypeError: if the response content type is not JSON.
273+
:raises ValueError: if the response content type is not JSON.
285274
:rtype: dict or str
286275
:returns: The API response payload, either as a raw string or
287276
a dictionary if the response is valid JSON.
@@ -296,21 +285,14 @@ def api_request(self, method, path, query_params=None,
296285
data = json.dumps(data)
297286
content_type = 'application/json'
298287

299-
response, content = self._make_request(
288+
response = self._make_request(
300289
method=method, url=url, data=data, content_type=content_type,
301290
headers=headers, target_object=_target_object)
302291

303-
if not 200 <= response.status < 300:
304-
raise make_exception(response, content,
305-
error_info=method + ' ' + url)
292+
if not 200 <= response.status_code < 300:
293+
raise exceptions.from_http_response(response)
306294

307-
string_or_bytes = (six.binary_type, six.text_type)
308-
if content and expect_json and isinstance(content, string_or_bytes):
309-
content_type = response.get('content-type', '')
310-
if not content_type.startswith('application/json'):
311-
raise TypeError('Expected JSON, got %s' % content_type)
312-
if isinstance(content, six.binary_type):
313-
content = content.decode('utf-8')
314-
return json.loads(content)
315-
316-
return content
295+
if expect_json and response.content:
296+
return response.json()
297+
else:
298+
return response.content

0 commit comments

Comments
 (0)