Package googleapiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.http

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actuall HTTP request.
      20  """
      21  from __future__ import absolute_import
      22  import six
      23  from six.moves import http_client
      24  from six.moves import range
      25
      26  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      27
      28  from six import BytesIO, StringIO
      29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
      30
      31  import base64
      32  import copy
      33  import gzip
      34  import httplib2
      35  import json
      36  import logging
      37  import mimetypes
      38  import os
      39  import random
      40  import socket
      41  import sys
      42  import time
      43  import uuid
      44
      45  # TODO(issue 221): Remove this conditional import jibbajabba.
      46  try:
      47    import ssl
      48  except ImportError:
      49    _ssl_SSLError = object()
      50  else:
      51    _ssl_SSLError = ssl.SSLError
      52
      53  from email.generator import Generator
      54  from email.mime.multipart import MIMEMultipart
      55  from email.mime.nonmultipart import MIMENonMultipart
      56  from email.parser import FeedParser
      57
      58  # Oauth2client < 3 has the positional helper in 'util', >= 3 has it
      59  # in '_helpers'.
      60  try:
      61    from oauth2client import util
      62  except ImportError:
      63    from oauth2client import _helpers as util
      64
      65  from googleapiclient import mimeparse
      66  from googleapiclient.errors import BatchError
      67  from googleapiclient.errors import HttpError
      68  from googleapiclient.errors import InvalidChunkSizeError
      69  from googleapiclient.errors import ResumableUploadError
      70  from googleapiclient.errors import UnexpectedBodyError
      71  from googleapiclient.errors import UnexpectedMethodError
      72  from googleapiclient.model import JsonModel
      73
      74
      75  LOGGER = logging.getLogger(__name__)
      76
      77  DEFAULT_CHUNK_SIZE = 512*1024
      78
      79  MAX_URI_LENGTH = 2048
      80
      81  _TOO_MANY_REQUESTS = 429
      82
      83
      84 -def _should_retry_response(resp_status, content):
      85    """Determines whether a response should be retried.
      86
      87    Args:
      88      resp_status: The response status received.
      89      content: The response content body.
      90
      91    Returns:
      92      True if the response should be retried, otherwise False.
      93    """
      94    # Retry on 5xx errors.
      95    if resp_status >= 500:
      96      return True
      97
      98    # Retry on 429 errors.
      99    if resp_status == _TOO_MANY_REQUESTS:
     100      return True
     101
     102    # For 403 errors, we have to check for the `reason` in the response to
     103    # determine if we should retry.
     104    if resp_status == six.moves.http_client.FORBIDDEN:
     105      # If there's no details about the 403 type, don't retry.
     106      if not content:
     107        return False
     108
     109      # Content is in JSON format.
     110      try:
     111        data = json.loads(content.decode('utf-8'))
     112        reason = data['error']['errors'][0]['reason']
     113      except (UnicodeDecodeError, ValueError, KeyError):
     114        LOGGER.warning('Invalid JSON content from response: %s', content)
     115        return False
     116
     117      LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
     118
     119      # Only retry on rate limit related failures.
     120      if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
     121        return True
     122
     123    # Everything else is a success or non-retriable so break.
     124    return False
     125
     126
     127 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
     128                     **kwargs):
     129    """Retries an HTTP request multiple times while handling errors.
     130
     131    If after all retries the request still fails, last error is either returned as
     132    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
     133
     134    Args:
     135      http: Http object to be used to execute request.
     136      num_retries: Maximum number of retries.
     137      req_type: Type of the request (used for logging retries).
     138      sleep, rand: Functions to sleep for random time between retries.
     139      uri: URI to be requested.
     140      method: HTTP method to be used.
     141      args, kwargs: Additional arguments passed to http.request.
     142
     143    Returns:
     144      resp, content - Response from the http request (may be HTTP 5xx).
     145    """
     146    resp = None
     147    content = None
     148    for retry_num in range(num_retries + 1):
     149      if retry_num > 0:
     150        # Sleep before retrying.
     151        sleep_time = rand() * 2 ** retry_num
     152        LOGGER.warning(
     153            'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
     154            sleep_time, retry_num, num_retries, req_type, method, uri,
     155            resp.status if resp else exception)
     156        sleep(sleep_time)
     157
     158      try:
     159        exception = None
     160        resp, content = http.request(uri, method, *args, **kwargs)
     161      # Retry on SSL errors and socket timeout errors.
     162      except _ssl_SSLError as ssl_error:
     163        exception = ssl_error
     164      except socket.error as socket_error:
     165        # errno's contents differ by platform, so we have to match by name.
     166        if socket.errno.errorcode.get(socket_error.errno) not in (
     167            'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
     168          raise
     169        exception = socket_error
     170
     171      if exception:
     172        if retry_num == num_retries:
     173          raise exception
     174        else:
     175          continue
     176
     177      if not _should_retry_response(resp.status, content):
     178        break
     179
     180    return resp, content
     181
     182
     183 -class MediaUploadProgress(object):
     184    """Status of a resumable upload."""
     185
     186 -  def __init__(self, resumable_progress, total_size):
     187      """Constructor.
     188
     189      Args:
     190        resumable_progress: int, bytes sent so far.
     191        total_size: int, total bytes in complete upload, or None if the total
     192          upload size isn't known ahead of time.
     193      """
     194      self.resumable_progress = resumable_progress
     195      self.total_size = total_size
     196
     197 -  def progress(self):
     198      """Percent of upload completed, as a float.
     199
     200      Returns:
     201        the percentage complete as a float, returning 0.0 if the total size of
     202        the upload is unknown.
     203      """
     204      if self.total_size is not None:
     205        return float(self.resumable_progress) / float(self.total_size)
     206      else:
     207        return 0.0
     208
     209
     210 -class MediaDownloadProgress(object):
     211    """Status of a resumable download."""
     212
     213 -  def __init__(self, resumable_progress, total_size):
     214      """Constructor.
     215
     216      Args:
     217        resumable_progress: int, bytes received so far.
     218        total_size: int, total bytes in complete download.
     219      """
     220      self.resumable_progress = resumable_progress
     221      self.total_size = total_size
     222
     223 -  def progress(self):
     224      """Percent of download completed, as a float.
     225
     226      Returns:
     227        the percentage complete as a float, returning 0.0 if the total size of
     228        the download is unknown.
     229      """
     230      if self.total_size is not None:
     231        return float(self.resumable_progress) / float(self.total_size)
     232      else:
     233        return 0.0
     234
     235
     236 -class MediaUpload(object):
     237    """Describes a media object to upload.
     238
     239    Base class that defines the interface of MediaUpload subclasses.
     240
     241    Note that subclasses of MediaUpload may allow you to control the chunksize
     242    when uploading a media object. It is important to keep the size of the chunk
     243    as large as possible to keep the upload efficient. Other factors may influence
     244    the size of the chunk you use, particularly if you are working in an
     245    environment where individual HTTP requests may have a hardcoded time limit,
     246    such as under certain classes of requests under Google App Engine.
     247
     248    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     249    subclasses support using streams directly to upload data. Support for
     250    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     251    platform that stream will be used for uploading the media object. The support
     252    for streaming is indicated by has_stream() returning True. The stream() method
     253    should return an io.Base object that supports seek(). On platforms where the
     254    underlying httplib module supports streaming, for example Python 2.6 and
     255    later, the stream will be passed into the http library which will result in
     256    less memory being used and possibly faster uploads.
     257
     258    If you need to upload media that can't be uploaded using any of the existing
     259    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     260    needs.
     261    """
     262
     263 -  def chunksize(self):
     264      """Chunk size for resumable uploads.
     265
     266      Returns:
     267        Chunk size in bytes.
     268      """
     269      raise NotImplementedError()
     270
     271 -  def mimetype(self):
     272      """Mime type of the body.
     273
     274      Returns:
     275        Mime type.
     276      """
     277      return 'application/octet-stream'
     278
     279 -  def size(self):
     280      """Size of upload.
     281
     282      Returns:
     283        Size of the body, or None of the size is unknown.
     284      """
     285      return None
     286
     287 -  def resumable(self):
     288      """Whether this upload is resumable.
     289
     290      Returns:
     291        True if resumable upload or False.
     292      """
     293      return False
     294
     295 -  def getbytes(self, begin, end):
     296      """Get bytes from the media.
     297
     298      Args:
     299        begin: int, offset from beginning of file.
     300        length: int, number of bytes to read, starting at begin.
     301
     302      Returns:
     303        A string of bytes read. May be shorter than length if EOF was reached
     304        first.
     305      """
     306      raise NotImplementedError()
     307
     308 -  def has_stream(self):
     309      """Does the underlying upload support a streaming interface.
     310
     311      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     312      seekable() returns True.
     313
     314      Returns:
     315        True if the call to stream() will return an instance of a seekable io.Base
     316        subclass.
     317      """
     318      return False
     319
     320 -  def stream(self):
     321      """A stream interface to the data being uploaded.
     322
     323      Returns:
     324        The returned value is an io.IOBase subclass that supports seek, i.e.
     325        seekable() returns True.
     326      """
     327      raise NotImplementedError()
     328
     329    @util.positional(1)
     330 -  def _to_json(self, strip=None):
     331      """Utility function for creating a JSON representation of a MediaUpload.
     332
     333      Args:
     334        strip: array, An array of names of members to not include in the JSON.
     335
     336      Returns:
     337         string, a JSON representation of this instance, suitable to pass to
     338         from_json().
     339      """
     340      t = type(self)
     341      d = copy.copy(self.__dict__)
     342      if strip is not None:
     343        for member in strip:
     344          del d[member]
     345      d['_class'] = t.__name__
     346      d['_module'] = t.__module__
     347      return json.dumps(d)
     348
     349 -  def to_json(self):
     350      """Create a JSON representation of an instance of MediaUpload.
     351
     352      Returns:
     353         string, a JSON representation of this instance, suitable to pass to
     354         from_json().
     355      """
     356      return self._to_json()
     357
     358    @classmethod
     359 -  def new_from_json(cls, s):
     360      """Utility class method to instantiate a MediaUpload subclass from a JSON
     361      representation produced by to_json().
     362
     363      Args:
     364        s: string, JSON from to_json().
     365
     366      Returns:
     367        An instance of the subclass of MediaUpload that was serialized with
     368        to_json().
     369      """
     370      data = json.loads(s)
     371      # Find and call the right classmethod from_json() to restore the object.
     372      module = data['_module']
     373      m = __import__(module, fromlist=module.split('.')[:-1])
     374      kls = getattr(m, data['_class'])
     375      from_json = getattr(kls, 'from_json')
     376      return from_json(s)
     377
     378
     379 -class MediaIoBaseUpload(MediaUpload):
     380    """A MediaUpload for a io.Base objects.
     381
     382    Note that the Python file object is compatible with io.Base and can be used
     383    with this class also.
     384
     385      fh = BytesIO('...Some data to upload...')
     386      media = MediaIoBaseUpload(fh, mimetype='image/png',
     387        chunksize=1024*1024, resumable=True)
     388      farm.animals().insert(
     389          id='cow',
     390          name='cow.png',
     391          media_body=media).execute()
     392
     393    Depending on the platform you are working on, you may pass -1 as the
     394    chunksize, which indicates that the entire file should be uploaded in a single
     395    request. If the underlying platform supports streams, such as Python 2.6 or
     396    later, then this can be very efficient as it avoids multiple connections, and
     397    also avoids loading the entire file into memory before sending it. Note that
     398    Google App Engine has a 5MB limit on request size, so you should never set
     399    your chunksize larger than 5MB, or to -1.
     400    """
     401
     402    @util.positional(3)
     403 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     404        resumable=False):
     405      """Constructor.
     406
     407      Args:
     408        fd: io.Base or file object, The source of the bytes to upload. MUST be
     409          opened in blocking mode, do not use streams opened in non-blocking mode.
     410          The given stream must be seekable, that is, it must be able to call
     411          seek() on fd.
     412        mimetype: string, Mime-type of the file.
     413        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     414          used if resumable=True. Pass in a value of -1 if the file is to be
     415          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     416          on request size, so you should never set your chunksize larger than 5MB,
     417          or to -1.
     418        resumable: bool, True if this is a resumable upload. False means upload
     419          in a single request.
     420      """
     421      super(MediaIoBaseUpload, self).__init__()
     422      self._fd = fd
     423      self._mimetype = mimetype
     424      if not (chunksize == -1 or chunksize > 0):
     425        raise InvalidChunkSizeError()
     426      self._chunksize = chunksize
     427      self._resumable = resumable
     428
     429      self._fd.seek(0, os.SEEK_END)
     430      self._size = self._fd.tell()
     431
     432 -  def chunksize(self):
     433      """Chunk size for resumable uploads.
     434
     435      Returns:
     436        Chunk size in bytes.
     437      """
     438      return self._chunksize
     439
     440 -  def mimetype(self):
     441      """Mime type of the body.
     442
     443      Returns:
     444        Mime type.
     445      """
     446      return self._mimetype
     447
     448 -  def size(self):
     449      """Size of upload.
     450
     451      Returns:
     452        Size of the body, or None of the size is unknown.
     453      """
     454      return self._size
     455
     456 -  def resumable(self):
     457      """Whether this upload is resumable.
     458
     459      Returns:
     460        True if resumable upload or False.
     461      """
     462      return self._resumable
     463
     464 -  def getbytes(self, begin, length):
     465      """Get bytes from the media.
     466
     467      Args:
     468        begin: int, offset from beginning of file.
     469        length: int, number of bytes to read, starting at begin.
     470
     471      Returns:
     472        A string of bytes read. May be shorted than length if EOF was reached
     473        first.
     474      """
     475      self._fd.seek(begin)
     476      return self._fd.read(length)
     477
     478 -  def has_stream(self):
     479      """Does the underlying upload support a streaming interface.
     480
     481      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     482      seekable() returns True.
     483
     484      Returns:
     485        True if the call to stream() will return an instance of a seekable io.Base
     486        subclass.
     487      """
     488      return True
     489
     490 -  def stream(self):
     491      """A stream interface to the data being uploaded.
     492
     493      Returns:
     494        The returned value is an io.IOBase subclass that supports seek, i.e.
     495        seekable() returns True.
     496      """
     497      return self._fd
     498
     499 -  def to_json(self):
     500      """This upload type is not serializable."""
     501      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     502
     503
     504 -class MediaFileUpload(MediaIoBaseUpload):
     505    """A MediaUpload for a file.
     506
     507    Construct a MediaFileUpload and pass as the media_body parameter of the
     508    method. For example, if we had a service that allowed uploading images:
     509
     510
     511      media = MediaFileUpload('cow.png', mimetype='image/png',
     512        chunksize=1024*1024, resumable=True)
     513      farm.animals().insert(
     514          id='cow',
     515          name='cow.png',
     516          media_body=media).execute()
     517
     518    Depending on the platform you are working on, you may pass -1 as the
     519    chunksize, which indicates that the entire file should be uploaded in a single
     520    request. If the underlying platform supports streams, such as Python 2.6 or
     521    later, then this can be very efficient as it avoids multiple connections, and
     522    also avoids loading the entire file into memory before sending it. Note that
     523    Google App Engine has a 5MB limit on request size, so you should never set
     524    your chunksize larger than 5MB, or to -1.
     525    """
     526
     527    @util.positional(2)
     528 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     529                 resumable=False):
     530      """Constructor.
     531
     532      Args:
     533        filename: string, Name of the file.
     534        mimetype: string, Mime-type of the file. If None then a mime-type will be
     535          guessed from the file extension.
     536        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     537          used if resumable=True. Pass in a value of -1 if the file is to be
     538          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     539          on request size, so you should never set your chunksize larger than 5MB,
     540          or to -1.
     541        resumable: bool, True if this is a resumable upload. False means upload
     542          in a single request.
     543      """
     544      self._filename = filename
     545      fd = open(self._filename, 'rb')
     546      if mimetype is None:
     547        # No mimetype provided, make a guess.
     548        mimetype, _ = mimetypes.guess_type(filename)
     549        if mimetype is None:
     550          # Guess failed, use octet-stream.
     551          mimetype = 'application/octet-stream'
     552      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     553                                            resumable=resumable)
     554
     555 -  def to_json(self):
     556      """Creating a JSON representation of an instance of MediaFileUpload.
     557
     558      Returns:
     559         string, a JSON representation of this instance, suitable to pass to
     560         from_json().
     561      """
     562      return self._to_json(strip=['_fd'])
     563
     564    @staticmethod
     565 -  def from_json(s):
     566      d = json.loads(s)
     567      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     568                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     569
     570
     571 -class MediaInMemoryUpload(MediaIoBaseUpload):
     572    """MediaUpload for a chunk of bytes.
     573
     574    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     575    the stream.
     576    """
     577
     578    @util.positional(2)
     579 -  def __init__(self, body, mimetype='application/octet-stream',
     580                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     581      """Create a new MediaInMemoryUpload.
     582
     583    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     584    the stream.
     585
     586    Args:
     587      body: string, Bytes of body content.
     588      mimetype: string, Mime-type of the file or default of
     589        'application/octet-stream'.
     590      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     591        used if resumable=True.
     592      resumable: bool, True if this is a resumable upload. False means upload
     593        in a single request.
     594      """
     595      fd = BytesIO(body)
     596      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     597                                                resumable=resumable)
     598
     599
     600 -class MediaIoBaseDownload(object):
     601    """"Download media resources.
     602
     603    Note that the Python file object is compatible with io.Base and can be used
     604    with this class also.
     605
     606
     607    Example:
     608      request = farms.animals().get_media(id='cow')
     609      fh = io.FileIO('cow.png', mode='wb')
     610      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     611
     612      done = False
     613      while done is False:
     614        status, done = downloader.next_chunk()
     615        if status:
     616          print "Download %d%%." % int(status.progress() * 100)
     617      print "Download Complete!"
     618    """
     619
     620    @util.positional(3)
     621 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     622      """Constructor.
     623
     624      Args:
     625        fd: io.Base or file object, The stream in which to write the downloaded
     626          bytes.
     627        request: googleapiclient.http.HttpRequest, the media request to perform in
     628          chunks.
     629        chunksize: int, File will be downloaded in chunks of this many bytes.
     630      """
     631      self._fd = fd
     632      self._request = request
     633      self._uri = request.uri
     634      self._chunksize = chunksize
     635      self._progress = 0
     636      self._total_size = None
     637      self._done = False
     638
     639      # Stubs for testing.
     640      self._sleep = time.sleep
     641      self._rand = random.random
     642
     643    @util.positional(1)
     644 -  def next_chunk(self, num_retries=0):
     645      """Get the next chunk of the download.
     646
     647      Args:
     648        num_retries: Integer, number of times to retry with randomized
     649              exponential backoff. If all retries fail, the raised HttpError
     650              represents the last request. If zero (default), we attempt the
     651              request only once.
     652
     653      Returns:
     654        (status, done): (MediaDownloadStatus, boolean)
     655           The value of 'done' will be True when the media has been fully
     656           downloaded.
     657
     658      Raises:
     659        googleapiclient.errors.HttpError if the response was not a 2xx.
     660        httplib2.HttpLib2Error if a transport error has occured.
     661      """
     662      headers = {
     663          'range': 'bytes=%d-%d' % (
     664              self._progress, self._progress + self._chunksize)
     665          }
     666      http = self._request.http
     667
     668      resp, content = _retry_request(
     669          http, num_retries, 'media download', self._sleep, self._rand, self._uri,
     670          'GET', headers=headers)
     671
     672      if resp.status in [200, 206]:
     673        if 'content-location' in resp and resp['content-location'] != self._uri:
     674          self._uri = resp['content-location']
     675        self._progress += len(content)
     676        self._fd.write(content)
     677
     678        if 'content-range' in resp:
     679          content_range = resp['content-range']
     680          length = content_range.rsplit('/', 1)[1]
     681          self._total_size = int(length)
     682        elif 'content-length' in resp:
     683          self._total_size = int(resp['content-length'])
     684
     685        if self._progress == self._total_size:
     686          self._done = True
     687        return MediaDownloadProgress(self._progress, self._total_size), self._done
     688      else:
     689        raise HttpError(resp, content, uri=self._uri)
     690
     691
     692 -class _StreamSlice(object):
     693    """Truncated stream.
     694
     695    Takes a stream and presents a stream that is a slice of the original stream.
     696    This is used when uploading media in chunks. In later versions of Python a
     697    stream can be passed to httplib in place of the string of data to send. The
     698    problem is that httplib just blindly reads to the end of the stream. This
     699    wrapper presents a virtual stream that only reads to the end of the chunk.
     700    """
     701
     702 -  def __init__(self, stream, begin, chunksize):
     703      """Constructor.
     704
     705      Args:
     706        stream: (io.Base, file object), the stream to wrap.
     707        begin: int, the seek position the chunk begins at.
     708        chunksize: int, the size of the chunk.
     709      """
     710      self._stream = stream
     711      self._begin = begin
     712      self._chunksize = chunksize
     713      self._stream.seek(begin)
     714
     715 -  def read(self, n=-1):
     716      """Read n bytes.
     717
     718      Args:
     719        n, int, the number of bytes to read.
     720
     721      Returns:
     722        A string of length 'n', or less if EOF is reached.
     723      """
     724      # The data left available to read sits in [cur, end)
     725      cur = self._stream.tell()
     726      end = self._begin + self._chunksize
     727      if n == -1 or cur + n > end:
     728        n = end - cur
     729      return self._stream.read(n)
     730
     731
     732 -class HttpRequest(object):
     733    """Encapsulates a single HTTP request."""
     734
     735    @util.positional(4)
     736 -  def __init__(self, http, postproc, uri,
     737                 method='GET',
     738                 body=None,
     739                 headers=None,
     740                 methodId=None,
     741                 resumable=None):
     742      """Constructor for an HttpRequest.
     743
     744      Args:
     745        http: httplib2.Http, the transport object to use to make a request
     746        postproc: callable, called on the HTTP response and content to transform
     747                  it into a data object before returning, or raising an exception
     748                  on an error.
     749        uri: string, the absolute URI to send the request to
     750        method: string, the HTTP method to use
     751        body: string, the request body of the HTTP request,
     752        headers: dict, the HTTP request headers
     753        methodId: string, a unique identifier for the API method being called.
     754        resumable: MediaUpload, None if this is not a resumbale request.
     755      """
     756      self.uri = uri
     757      self.method = method
     758      self.body = body
     759      self.headers = headers or {}
     760      self.methodId = methodId
     761      self.http = http
     762      self.postproc = postproc
     763      self.resumable = resumable
     764      self.response_callbacks = []
     765      self._in_error_state = False
     766
     767      # Pull the multipart boundary out of the content-type header.
     768      major, minor, params = mimeparse.parse_mime_type(
     769          self.headers.get('content-type', 'application/json'))
     770
     771      # The size of the non-media part of the request.
     772      self.body_size = len(self.body or '')
     773
     774      # The resumable URI to send chunks to.
     775      self.resumable_uri = None
     776
     777      # The bytes that have been uploaded.
     778      self.resumable_progress = 0
     779
     780      # Stubs for testing.
     781      self._rand = random.random
     782      self._sleep = time.sleep
     783
     784    @util.positional(1)
     785 -  def execute(self, http=None, num_retries=0):
     786      """Execute the request.
     787
     788      Args:
     789        http: httplib2.Http, an http object to be used in place of the
     790              one the HttpRequest request object was constructed with.
     791        num_retries: Integer, number of times to retry with randomized
     792              exponential backoff. If all retries fail, the raised HttpError
     793              represents the last request. If zero (default), we attempt the
     794              request only once.
     795
     796      Returns:
     797        A deserialized object model of the response body as determined
     798        by the postproc.
     799
     800      Raises:
     801        googleapiclient.errors.HttpError if the response was not a 2xx.
     802        httplib2.HttpLib2Error if a transport error has occured.
     803      """
     804      if http is None:
     805        http = self.http
     806
     807      if self.resumable:
     808        body = None
     809        while body is None:
     810          _, body = self.next_chunk(http=http, num_retries=num_retries)
     811        return body
     812
     813      # Non-resumable case.
     814
     815      if 'content-length' not in self.headers:
     816        self.headers['content-length'] = str(self.body_size)
     817      # If the request URI is too long then turn it into a POST request.
     818      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     819        self.method = 'POST'
     820        self.headers['x-http-method-override'] = 'GET'
     821        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     822        parsed = urlparse(self.uri)
     823        self.uri = urlunparse(
     824            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     825             None)
     826            )
     827        self.body = parsed.query
     828        self.headers['content-length'] = str(len(self.body))
     829
     830      # Handle retries for server-side errors.
     831      resp, content = _retry_request(
     832            http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
     833            method=str(self.method), body=self.body, headers=self.headers)
     834
     835      for callback in self.response_callbacks:
     836        callback(resp)
     837      if resp.status >= 300:
     838        raise HttpError(resp, content, uri=self.uri)
     839      return self.postproc(resp, content)
     840
     841    @util.positional(2)
     842 -  def add_response_callback(self, cb):
     843      """add_response_headers_callback
     844
     845      Args:
     846        cb: Callback to be called on receiving the response headers, of signature:
     847
     848        def cb(resp):
     849          # Where resp is an instance of httplib2.Response
     850      """
     851      self.response_callbacks.append(cb)
     852
     853    @util.positional(1)
     854 -  def next_chunk(self, http=None, num_retries=0):
     855      """Execute the next step of a resumable upload.
     856
     857      Can only be used if the method being executed supports media uploads and
     858      the MediaUpload object passed in was flagged as using resumable upload.
     859
     860      Example:
     861
     862        media = MediaFileUpload('cow.png', mimetype='image/png',
     863                                chunksize=1000, resumable=True)
     864        request = farm.animals().insert(
     865            id='cow',
     866            name='cow.png',
     867            media_body=media)
     868
     869        response = None
     870        while response is None:
     871          status, response = request.next_chunk()
     872          if status:
     873            print "Upload %d%% complete." % int(status.progress() * 100)
     874
     875
     876      Args:
     877        http: httplib2.Http, an http object to be used in place of the
     878              one the HttpRequest request object was constructed with.
     879        num_retries: Integer, number of times to retry with randomized
     880              exponential backoff. If all retries fail, the raised HttpError
     881              represents the last request. If zero (default), we attempt the
     882              request only once.
     883
     884      Returns:
     885        (status, body): (ResumableMediaStatus, object)
     886           The body will be None until the resumable media is fully uploaded.
     887
     888      Raises:
     889        googleapiclient.errors.HttpError if the response was not a 2xx.
     890        httplib2.HttpLib2Error if a transport error has occured.
     891      """
     892      if http is None:
     893        http = self.http
     894
     895      if self.resumable.size() is None:
     896        size = '*'
     897      else:
     898        size = str(self.resumable.size())
     899
     900      if self.resumable_uri is None:
     901        start_headers = copy.copy(self.headers)
     902        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     903        if size != '*':
     904          start_headers['X-Upload-Content-Length'] = size
     905        start_headers['content-length'] = str(self.body_size)
     906
     907        resp, content = _retry_request(
     908            http, num_retries, 'resumable URI request', self._sleep, self._rand,
     909            self.uri, method=self.method, body=self.body, headers=start_headers)
     910
     911        if resp.status == 200 and 'location' in resp:
     912          self.resumable_uri = resp['location']
     913        else:
     914          raise ResumableUploadError(resp, content)
     915      elif self._in_error_state:
     916        # If we are in an error state then query the server for current state of
     917        # the upload by sending an empty PUT and reading the 'range' header in
     918        # the response.
     919        headers = {
     920            'Content-Range': 'bytes */%s' % size,
     921            'content-length': '0'
     922            }
     923        resp, content = http.request(self.resumable_uri, 'PUT',
     924                                     headers=headers)
     925        status, body = self._process_response(resp, content)
     926        if body:
     927          # The upload was complete.
     928          return (status, body)
     929
     930      if self.resumable.has_stream():
     931        data = self.resumable.stream()
     932        if self.resumable.chunksize() == -1:
     933          data.seek(self.resumable_progress)
     934          chunk_end = self.resumable.size() - self.resumable_progress - 1
     935        else:
     936          # Doing chunking with a stream, so wrap a slice of the stream.
     937          data = _StreamSlice(data, self.resumable_progress,
     938                              self.resumable.chunksize())
     939          chunk_end = min(
     940              self.resumable_progress + self.resumable.chunksize() - 1,
     941              self.resumable.size() - 1)
     942      else:
     943        data = self.resumable.getbytes(
     944            self.resumable_progress, self.resumable.chunksize())
     945
     946        # A short read implies that we are at EOF, so finish the upload.
     947        if len(data) < self.resumable.chunksize():
     948          size = str(self.resumable_progress + len(data))
     949
     950        chunk_end = self.resumable_progress + len(data) - 1
     951
     952      headers = {
     953          'Content-Range': 'bytes %d-%d/%s' % (
     954              self.resumable_progress, chunk_end, size),
     955          # Must set the content-length header here because httplib can't
     956          # calculate the size when working with _StreamSlice.
     957          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     958          }
     959
     960      for retry_num in range(num_retries + 1):
     961        if retry_num > 0:
     962          self._sleep(self._rand() * 2**retry_num)
     963          LOGGER.warning(
     964              'Retry #%d for media upload: %s %s, following status: %d'
     965              % (retry_num, self.method, self.uri, resp.status))
     966
     967        try:
     968          resp, content = http.request(self.resumable_uri, method='PUT',
     969                                       body=data,
     970                                       headers=headers)
     971        except:
     972          self._in_error_state = True
     973          raise
     974        if not _should_retry_response(resp.status, content):
     975          break
     976
     977      return self._process_response(resp, content)
     978
     979 -  def _process_response(self, resp, content):
     980      """Process the response from a single chunk upload.
     981
     982      Args:
     983        resp: httplib2.Response, the response object.
     984        content: string, the content of the response.
     985
     986      Returns:
     987        (status, body): (ResumableMediaStatus, object)
     988           The body will be None until the resumable media is fully uploaded.
     989
     990      Raises:
     991        googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
     992      """
     993      if resp.status in [200, 201]:
     994        self._in_error_state = False
     995        return None, self.postproc(resp, content)
     996      elif resp.status == 308:
     997        self._in_error_state = False
     998        # A "308 Resume Incomplete" indicates we are not done.
     999        self.resumable_progress = int(resp['range'].split('-')[1]) + 1
    1000        if 'location' in resp:
    1001          self.resumable_uri = resp['location']
    1002      else:
    1003        self._in_error_state = True
    1004        raise HttpError(resp, content, uri=self.uri)
    1005
    1006      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
    1007              None)
    1008
    1009 -  def to_json(self):
    1010      """Returns a JSON representation of the HttpRequest."""
    1011      d = copy.copy(self.__dict__)
    1012      if d['resumable'] is not None:
    1013        d['resumable'] = self.resumable.to_json()
    1014      del d['http']
    1015      del d['postproc']
    1016      del d['_sleep']
    1017      del d['_rand']
    1018
    1019      return json.dumps(d)
    1020
    1021    @staticmethod
    1022 -  def from_json(s, http, postproc):
    1023      """Returns an HttpRequest populated with info from a JSON object."""
    1024      d = json.loads(s)
    1025      if d['resumable'] is not None:
    1026        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
    1027      return HttpRequest(
    1028          http,
    1029          postproc,
    1030          uri=d['uri'],
    1031          method=d['method'],
    1032          body=d['body'],
    1033          headers=d['headers'],
    1034          methodId=d['methodId'],
    1035          resumable=d['resumable'])
    1036
    1037
    1038 -class BatchHttpRequest(object):
    1039    """Batches multiple HttpRequest objects into a single HTTP request.
    1040
    1041    Example:
    1042      from googleapiclient.http import BatchHttpRequest
    1043
    1044      def list_animals(request_id, response, exception):
    1045        \"\"\"Do something with the animals list response.\"\"\"
    1046        if exception is not None:
    1047          # Do something with the exception.
    1048          pass
    1049        else:
    1050          # Do something with the response.
    1051          pass
    1052
    1053      def list_farmers(request_id, response, exception):
    1054        \"\"\"Do something with the farmers list response.\"\"\"
    1055        if exception is not None:
    1056          # Do something with the exception.
    1057          pass
    1058        else:
    1059          # Do something with the response.
    1060          pass
    1061
    1062      service = build('farm', 'v2')
    1063
    1064      batch = BatchHttpRequest()
    1065
    1066      batch.add(service.animals().list(), list_animals)
    1067      batch.add(service.farmers().list(), list_farmers)
    1068      batch.execute(http=http)
    1069    """
    1070
    1071    @util.positional(1)
    1072 -  def __init__(self, callback=None, batch_uri=None):
    1073      """Constructor for a BatchHttpRequest.
    1074
    1075      Args:
    1076        callback: callable, A callback to be called for each response, of the
    1077          form callback(id, response, exception). The first parameter is the
    1078          request id, and the second is the deserialized response object. The
    1079          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1080          occurred while processing the request, or None if no error occurred.
    1081        batch_uri: string, URI to send batch requests to.
    1082      """
    1083      if batch_uri is None:
    1084        batch_uri = 'https://www.googleapis.com/batch'
    1085      self._batch_uri = batch_uri
    1086
    1087      # Global callback to be called for each individual response in the batch.
    1088      self._callback = callback
    1089
    1090      # A map from id to request.
    1091      self._requests = {}
    1092
    1093      # A map from id to callback.
    1094      self._callbacks = {}
    1095
    1096      # List of request ids, in the order in which they were added.
    1097      self._order = []
    1098
    1099      # The last auto generated id.
    1100      self._last_auto_id = 0
    1101
    1102      # Unique ID on which to base the Content-ID headers.
    1103      self._base_id = None
    1104
    1105      # A map from request id to (httplib2.Response, content) response pairs
    1106      self._responses = {}
    1107
    1108      # A map of id(Credentials) that have been refreshed.
    1109      self._refreshed_credentials = {}
    1110
    1111 -  def _refresh_and_apply_credentials(self, request, http):
    1112      """Refresh the credentials and apply to the request.
    1113
    1114      Args:
    1115        request: HttpRequest, the request.
    1116        http: httplib2.Http, the global http object for the batch.
    1117      """
    1118      # For the credentials to refresh, but only once per refresh_token
    1119      # If there is no http per the request then refresh the http passed in
    1120      # via execute()
    1121      creds = None
    1122      if request.http is not None and hasattr(request.http.request,
    1123          'credentials'):
    1124        creds = request.http.request.credentials
    1125      elif http is not None and hasattr(http.request, 'credentials'):
    1126        creds = http.request.credentials
    1127      if creds is not None:
    1128        if id(creds) not in self._refreshed_credentials:
    1129          creds.refresh(http)
    1130          self._refreshed_credentials[id(creds)] = 1
    1131
    1132      # Only apply the credentials if we are using the http object passed in,
    1133      # otherwise apply() will get called during _serialize_request().
    1134      if request.http is None or not hasattr(request.http.request,
    1135          'credentials'):
    1136        creds.apply(request.headers)
    1137
    1138 -  def _id_to_header(self, id_):
    1139      """Convert an id to a Content-ID header value.
    1140
    1141      Args:
    1142        id_: string, identifier of individual request.
    1143
    1144      Returns:
    1145        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1146        the value because Content-ID headers are supposed to be universally
    1147        unique.
    1148      """
    1149      if self._base_id is None:
    1150        self._base_id = uuid.uuid4()
    1151
    1152      return '<%s+%s>' % (self._base_id, quote(id_))
    1153
    1154 -  def _header_to_id(self, header):
    1155      """Convert a Content-ID header value to an id.
    1156
    1157      Presumes the Content-ID header conforms to the format that _id_to_header()
    1158      returns.
    1159
    1160      Args:
    1161        header: string, Content-ID header value.
    1162
    1163      Returns:
    1164        The extracted id value.
    1165
    1166      Raises:
    1167        BatchError if the header is not in the expected format.
    1168      """
    1169      if header[0] != '<' or header[-1] != '>':
    1170        raise BatchError("Invalid value for Content-ID: %s" % header)
    1171      if '+' not in header:
    1172        raise BatchError("Invalid value for Content-ID: %s" % header)
    1173      base, id_ = header[1:-1].rsplit('+', 1)
    1174
    1175      return unquote(id_)
    1176
    1177 -  def _serialize_request(self, request):
    1178      """Convert an HttpRequest object into a string.
    1179
    1180      Args:
    1181        request: HttpRequest, the request to serialize.
    1182
    1183      Returns:
    1184        The request as a string in application/http format.
    1185      """
    1186      # Construct status line
    1187      parsed = urlparse(request.uri)
    1188      request_line = urlunparse(
    1189          ('', '', parsed.path, parsed.params, parsed.query, '')
    1190          )
    1191      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1192      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1193      msg = MIMENonMultipart(major, minor)
    1194      headers = request.headers.copy()
    1195
    1196      if request.http is not None and hasattr(request.http.request,
    1197          'credentials'):
    1198        request.http.request.credentials.apply(headers)
    1199
    1200      # MIMENonMultipart adds its own Content-Type header.
    1201      if 'content-type' in headers:
    1202        del headers['content-type']
    1203
    1204      for key, value in six.iteritems(headers):
    1205        msg[key] = value
    1206      msg['Host'] = parsed.netloc
    1207      msg.set_unixfrom(None)
    1208
    1209      if request.body is not None:
    1210        msg.set_payload(request.body)
    1211        msg['content-length'] = str(len(request.body))
    1212
    1213      # Serialize the mime message.
    1214      fp = StringIO()
    1215      # maxheaderlen=0 means don't line wrap headers.
    1216      g = Generator(fp, maxheaderlen=0)
    1217      g.flatten(msg, unixfrom=False)
    1218      body = fp.getvalue()
    1219
    1220      return status_line + body
    1221
    1222 -  def _deserialize_response(self, payload):
    1223      """Convert string into httplib2 response and content.
    1224
    1225      Args:
    1226        payload: string, headers and body as a string.
    1227
    1228      Returns:
    1229        A pair (resp, content), such as would be returned from httplib2.request.
    1230      """
    1231      # Strip off the status line
    1232      status_line, payload = payload.split('\n', 1)
    1233      protocol, status, reason = status_line.split(' ', 2)
    1234
    1235      # Parse the rest of the response
    1236      parser = FeedParser()
    1237      parser.feed(payload)
    1238      msg = parser.close()
    1239      msg['status'] = status
    1240
    1241      # Create httplib2.Response from the parsed headers.
    1242      resp = httplib2.Response(msg)
    1243      resp.reason = reason
    1244      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1245
    1246      content = payload.split('\r\n\r\n', 1)[1]
    1247
    1248      return resp, content
    1249
    1250 -  def _new_id(self):
    1251      """Create a new id.
    1252
    1253      Auto incrementing number that avoids conflicts with ids already used.
    1254
    1255      Returns:
    1256         string, a new unique id.
    1257      """
    1258      self._last_auto_id += 1
    1259      while str(self._last_auto_id) in self._requests:
    1260        self._last_auto_id += 1
    1261      return str(self._last_auto_id)
    1262
    1263    @util.positional(2)
    1264 -  def add(self, request, callback=None, request_id=None):
    1265      """Add a new request.
    1266
    1267      Every callback added will be paired with a unique id, the request_id. That
    1268      unique id will be passed back to the callback when the response comes back
    1269      from the server. The default behavior is to have the library generate it's
    1270      own unique id. If the caller passes in a request_id then they must ensure
    1271      uniqueness for each request_id, and if they are not an exception is
    1272      raised. Callers should either supply all request_ids or nevery supply a
    1273      request id, to avoid such an error.
    1274
    1275      Args:
    1276        request: HttpRequest, Request to add to the batch.
    1277        callback: callable, A callback to be called for this response, of the
    1278          form callback(id, response, exception). The first parameter is the
    1279          request id, and the second is the deserialized response object. The
    1280          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1281          occurred while processing the request, or None if no errors occurred.
    1282        request_id: string, A unique id for the request. The id will be passed to
    1283          the callback with the response.
    1284
    1285      Returns:
    1286        None
    1287
    1288      Raises:
    1289        BatchError if a media request is added to a batch.
    1290        KeyError is the request_id is not unique.
    1291      """
    1292      if request_id is None:
    1293        request_id = self._new_id()
    1294      if request.resumable is not None:
    1295        raise BatchError("Media requests cannot be used in a batch request.")
    1296      if request_id in self._requests:
    1297        raise KeyError("A request with this ID already exists: %s" % request_id)
    1298      self._requests[request_id] = request
    1299      self._callbacks[request_id] = callback
    1300      self._order.append(request_id)
    1301
    1302 -  def _execute(self, http, order, requests):
    1303      """Serialize batch request, send to server, process response.
    1304
    1305      Args:
    1306        http: httplib2.Http, an http object to be used to make the request with.
    1307        order: list, list of request ids in the order they were added to the
    1308          batch.
    1309        request: list, list of request objects to send.
    1310
    1311      Raises:
    1312        httplib2.HttpLib2Error if a transport error has occured.
    1313        googleapiclient.errors.BatchError if the response is the wrong format.
    1314      """
    1315      message = MIMEMultipart('mixed')
    1316      # Message should not write out it's own headers.
    1317      setattr(message, '_write_headers', lambda self: None)
    1318
    1319      # Add all the individual requests.
    1320      for request_id in order:
    1321        request = requests[request_id]
    1322
    1323        msg = MIMENonMultipart('application', 'http')
    1324        msg['Content-Transfer-Encoding'] = 'binary'
    1325        msg['Content-ID'] = self._id_to_header(request_id)
    1326
    1327        body = self._serialize_request(request)
    1328        msg.set_payload(body)
    1329        message.attach(msg)
    1330
    1331      # encode the body: note that we can't use `as_string`, because
    1332      # it plays games with `From ` lines.
    1333      fp = StringIO()
    1334      g = Generator(fp, mangle_from_=False)
    1335      g.flatten(message, unixfrom=False)
    1336      body = fp.getvalue()
    1337
    1338      headers = {}
    1339      headers['content-type'] = ('multipart/mixed; '
    1340                                 'boundary="%s"') % message.get_boundary()
    1341
    1342      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1343                                   headers=headers)
    1344
    1345      if resp.status >= 300:
    1346        raise HttpError(resp, content, uri=self._batch_uri)
    1347
    1348      # Prepend with a content-type header so FeedParser can handle it.
    1349      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1350      # PY3's FeedParser only accepts unicode. So we should decode content
    1351      # here, and encode each payload again.
    1352      if six.PY3:
    1353        content = content.decode('utf-8')
    1354      for_parser = header + content
    1355
    1356      parser = FeedParser()
    1357      parser.feed(for_parser)
    1358      mime_response = parser.close()
    1359
    1360      if not mime_response.is_multipart():
    1361        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1362                         content=content)
    1363
    1364      for part in mime_response.get_payload():
    1365        request_id = self._header_to_id(part['Content-ID'])
    1366        response, content = self._deserialize_response(part.get_payload())
    1367        # We encode content here to emulate normal http response.
    1368        if isinstance(content, six.text_type):
    1369          content = content.encode('utf-8')
    1370        self._responses[request_id] = (response, content)
    1371
    1372    @util.positional(1)
    1373 -  def execute(self, http=None):
    1374      """Execute all the requests as a single batched HTTP request.
    1375
    1376      Args:
    1377        http: httplib2.Http, an http object to be used in place of the one the
    1378          HttpRequest request object was constructed with. If one isn't supplied
    1379          then use a http object from the requests in this batch.
    1380
    1381      Returns:
    1382        None
    1383
    1384      Raises:
    1385        httplib2.HttpLib2Error if a transport error has occured.
    1386        googleapiclient.errors.BatchError if the response is the wrong format.
    1387      """
    1388      # If we have no requests return
    1389      if len(self._order) == 0:
    1390        return None
    1391
    1392      # If http is not supplied use the first valid one given in the requests.
    1393      if http is None:
    1394        for request_id in self._order:
    1395          request = self._requests[request_id]
    1396          if request is not None:
    1397            http = request.http
    1398            break
    1399
    1400      if http is None:
    1401        raise ValueError("Missing a valid http object.")
    1402
    1403      # Special case for OAuth2Credentials-style objects which have not yet been
    1404      # refreshed with an initial access_token.
    1405      if getattr(http.request, 'credentials', None) is not None:
    1406        creds = http.request.credentials
    1407        if not getattr(creds, 'access_token', None):
    1408          LOGGER.info('Attempting refresh to obtain initial access_token')
    1409          creds.refresh(http)
    1410
    1411      self._execute(http, self._order, self._requests)
    1412
    1413      # Loop over all the requests and check for 401s. For each 401 request the
    1414      # credentials should be refreshed and then sent again in a separate batch.
    1415      redo_requests = {}
    1416      redo_order = []
    1417
    1418      for request_id in self._order:
    1419        resp, content = self._responses[request_id]
    1420        if resp['status'] == '401':
    1421          redo_order.append(request_id)
    1422          request = self._requests[request_id]
    1423          self._refresh_and_apply_credentials(request, http)
    1424          redo_requests[request_id] = request
    1425
    1426      if redo_requests:
    1427        self._execute(http, redo_order, redo_requests)
    1428
    1429      # Now process all callbacks that are erroring, and raise an exception for
    1430      # ones that return a non-2xx response? Or add extra parameter to callback
    1431      # that contains an HttpError?
    1432
    1433      for request_id in self._order:
    1434        resp, content = self._responses[request_id]
    1435
    1436        request = self._requests[request_id]
    1437        callback = self._callbacks[request_id]
    1438
    1439        response = None
    1440        exception = None
    1441        try:
    1442          if resp.status >= 300:
    1443            raise HttpError(resp, content, uri=request.uri)
    1444          response = request.postproc(resp, content)
    1445        except HttpError as e:
    1446          exception = e
    1447
    1448        if callback is not None:
    1449          callback(request_id, response, exception)
    1450        if self._callback is not None:
    1451          self._callback(request_id, response, exception)
    1452
    1453
    1454 -class HttpRequestMock(object):
    1455    """Mock of HttpRequest.
    1456
    1457    Do not construct directly, instead use RequestMockBuilder.
    1458    """
    1459
    1460 -  def __init__(self, resp, content, postproc):
    1461      """Constructor for HttpRequestMock
    1462
    1463      Args:
    1464        resp: httplib2.Response, the response to emulate coming from the request
    1465        content: string, the response body
    1466        postproc: callable, the post processing function usually supplied by
    1467                  the model class. See model.JsonModel.response() as an example.
    1468      """
    1469      self.resp = resp
    1470      self.content = content
    1471      self.postproc = postproc
    1472      if resp is None:
    1473        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1474      if 'reason' in self.resp:
    1475        self.resp.reason = self.resp['reason']
    1476
    1477 -  def execute(self, http=None):
    1478      """Execute the request.
    1479
    1480      Same behavior as HttpRequest.execute(), but the response is
    1481      mocked and not really from an HTTP request/response.
    1482      """
    1483      return self.postproc(self.resp, self.content)
    1484
    1485
    1486 -class RequestMockBuilder(object):
    1487    """A simple mock of HttpRequest
    1488
    1489      Pass in a dictionary to the constructor that maps request methodIds to
    1490      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1491      returned when that method is called. None may also be passed in for the
    1492      httplib2.Response, in which case a 200 OK response will be generated.
    1493      If an opt_expected_body (str or dict) is provided, it will be compared to
    1494      the body and UnexpectedBodyError will be raised on inequality.
    1495
    1496      Example:
    1497        response = '{"data": {"id": "tag:google.c...'
    1498        requestBuilder = RequestMockBuilder(
    1499          {
    1500            'plus.activities.get': (None, response),
    1501          }
    1502        )
    1503        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1504
    1505      Methods that you do not supply a response for will return a
    1506      200 OK with an empty string as the response content or raise an excpetion
    1507      if check_unexpected is set to True. The methodId is taken from the rpcName
    1508      in the discovery document.
    1509
    1510      For more details see the project wiki.
    1511    """
    1512
    1513 -  def __init__(self, responses, check_unexpected=False):
    1514      """Constructor for RequestMockBuilder
    1515
    1516      The constructed object should be a callable object
    1517      that can replace the class HttpResponse.
    1518
    1519      responses - A dictionary that maps methodIds into tuples
    1520                  of (httplib2.Response, content). The methodId
    1521                  comes from the 'rpcName' field in the discovery
    1522                  document.
    1523      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1524                         should be raised on unsupplied method.
    1525      """
    1526      self.responses = responses
    1527      self.check_unexpected = check_unexpected
    1528
    1529 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1530                 headers=None, methodId=None, resumable=None):
    1531      """Implements the callable interface that discovery.build() expects
    1532      of requestBuilder, which is to build an object compatible with
    1533      HttpRequest.execute(). See that method for the description of the
    1534      parameters and the expected response.
    1535      """
    1536      if methodId in self.responses:
    1537        response = self.responses[methodId]
    1538        resp, content = response[:2]
    1539        if len(response) > 2:
    1540          # Test the body against the supplied expected_body.
    1541          expected_body = response[2]
    1542          if bool(expected_body) != bool(body):
    1543            # Not expecting a body and provided one
    1544            # or expecting a body and not provided one.
    1545            raise UnexpectedBodyError(expected_body, body)
    1546          if isinstance(expected_body, str):
    1547            expected_body = json.loads(expected_body)
    1548          body = json.loads(body)
    1549          if body != expected_body:
    1550            raise UnexpectedBodyError(expected_body, body)
    1551        return HttpRequestMock(resp, content, postproc)
    1552      elif self.check_unexpected:
    1553        raise UnexpectedMethodError(methodId=methodId)
    1554      else:
    1555        model = JsonModel(False)
    1556        return HttpRequestMock(None, '{}', model.response)
    1557
    1558
    1559 -class HttpMock(object):
    1560    """Mock of httplib2.Http"""
    1561
    1562 -  def __init__(self, filename=None, headers=None):
    1563      """
    1564      Args:
    1565        filename: string, absolute filename to read response from
    1566        headers: dict, header to return with response
    1567      """
    1568      if headers is None:
    1569        headers = {'status': '200'}
    1570      if filename:
    1571        f = open(filename, 'rb')
    1572        self.data = f.read()
    1573        f.close()
    1574      else:
    1575        self.data = None
    1576      self.response_headers = headers
    1577      self.headers = None
    1578      self.uri = None
    1579      self.method = None
    1580      self.body = None
    1581      self.headers = None
    1582
    1583
    1584 -  def request(self, uri,
    1585                method='GET',
    1586                body=None,
    1587                headers=None,
    1588                redirections=1,
    1589                connection_type=None):
    1590      self.uri = uri
    1591      self.method = method
    1592      self.body = body
    1593      self.headers = headers
    1594      return httplib2.Response(self.response_headers), self.data
    1595
    1596
    1597 -class HttpMockSequence(object):
    1598    """Mock of httplib2.Http
    1599
    1600    Mocks a sequence of calls to request returning different responses for each
    1601    call. Create an instance initialized with the desired response headers
    1602    and content and then use as if an httplib2.Http instance.
    1603
    1604      http = HttpMockSequence([
    1605        ({'status': '401'}, ''),
    1606        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1607        ({'status': '200'}, 'echo_request_headers'),
    1608        ])
    1609      resp, content = http.request("http://examples.com")
    1610
    1611    There are special values you can pass in for content to trigger
    1612    behavours that are helpful in testing.
    1613
    1614    'echo_request_headers' means return the request headers in the response body
    1615    'echo_request_headers_as_json' means return the request headers in
    1616       the response body
    1617    'echo_request_body' means return the request body in the response body
    1618    'echo_request_uri' means return the request uri in the response body
    1619    """
    1620
    1621 -  def __init__(self, iterable):
    1622      """
    1623      Args:
    1624        iterable: iterable, a sequence of pairs of (headers, body)
    1625      """
    1626      self._iterable = iterable
    1627      self.follow_redirects = True
    1628
    1629 -  def request(self, uri,
    1630                method='GET',
    1631                body=None,
    1632                headers=None,
    1633                redirections=1,
    1634                connection_type=None):
    1635      resp, content = self._iterable.pop(0)
    1636      if content == 'echo_request_headers':
    1637        content = headers
    1638      elif content == 'echo_request_headers_as_json':
    1639        content = json.dumps(headers)
    1640      elif content == 'echo_request_body':
    1641        if hasattr(body, 'read'):
    1642          content = body.read()
    1643        else:
    1644          content = body
    1645      elif content == 'echo_request_uri':
    1646        content = uri
    1647      if isinstance(content, six.text_type):
    1648        content = content.encode('utf-8')
    1649      return httplib2.Response(resp), content
    1650
    1651
    1652 -def set_user_agent(http, user_agent):
    1653    """Set the user-agent on every request.
    1654
    1655    Args:
    1656       http - An instance of httplib2.Http
    1657           or something that acts like it.
    1658       user_agent: string, the value for the user-agent header.
    1659
    1660    Returns:
    1661       A modified instance of http that was passed in.
    1662
    1663    Example:
    1664
    1665      h = httplib2.Http()
    1666      h = set_user_agent(h, "my-app-name/6.0")
    1667
    1668    Most of the time the user-agent will be set doing auth, this is for the rare
    1669    cases where you are accessing an unauthenticated endpoint.
    1670    """
    1671    request_orig = http.request
    1672
    1673    # The closure that will replace 'httplib2.Http.request'.
    1674    def new_request(uri, method='GET', body=None, headers=None,
    1675                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1676                    connection_type=None):
    1677      """Modify the request headers to add the user-agent."""
    1678      if headers is None:
    1679        headers = {}
    1680      if 'user-agent' in headers:
    1681        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1682      else:
    1683        headers['user-agent'] = user_agent
    1684      resp, content = request_orig(uri, method, body, headers,
    1685                          redirections, connection_type)
    1686      return resp, content
    1687
    1688    http.request = new_request
    1689    return http
    1690
    1691
    1692 -def tunnel_patch(http):
    1693    """Tunnel PATCH requests over POST.
    1694    Args:
    1695       http - An instance of httplib2.Http
    1696           or something that acts like it.
    1697
    1698    Returns:
    1699       A modified instance of http that was passed in.
    1700
    1701    Example:
    1702
    1703      h = httplib2.Http()
    1704      h = tunnel_patch(h, "my-app-name/6.0")
    1705
    1706    Useful if you are running on a platform that doesn't support PATCH.
    1707    Apply this last if you are using OAuth 1.0, as changing the method
    1708    will result in a different signature.
    1709    """
    1710    request_orig = http.request
    1711
    1712    # The closure that will replace 'httplib2.Http.request'.
    1713    def new_request(uri, method='GET', body=None, headers=None,
    1714                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1715                    connection_type=None):
    1716      """Modify the request headers to add the user-agent."""
    1717      if headers is None:
    1718        headers = {}
    1719      if method == 'PATCH':
    1720        if 'oauth_token' in headers.get('authorization', ''):
    1721          LOGGER.warning(
    1722              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1723        headers['x-http-method-override'] = "PATCH"
    1724        method = 'POST'
    1725      resp, content = request_orig(uri, method, body, headers,
    1726                          redirections, connection_type)
    1727      return resp, content
    1728
    1729    http.request = new_request
    1730    return http
    1731