Package googleapiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.http

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actuall HTTP request.
      20  """
      21
      22  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      23
      24  import StringIO
      25  import base64
      26  import copy
      27  import gzip
      28  import httplib2
      29  import json
      30  import logging
      31  import mimeparse
      32  import mimetypes
      33  import os
      34  import random
      35  import sys
      36  import time
      37  import urllib
      38  import urlparse
      39  import uuid
      40
      41  from email.generator import Generator
      42  from email.mime.multipart import MIMEMultipart
      43  from email.mime.nonmultipart import MIMENonMultipart
      44  from email.parser import FeedParser
      45  from errors import BatchError
      46  from errors import HttpError
      47  from errors import InvalidChunkSizeError
      48  from errors import ResumableUploadError
      49  from errors import UnexpectedBodyError
      50  from errors import UnexpectedMethodError
      51  from model import JsonModel
      52  from oauth2client import util
      53
      54
      55  DEFAULT_CHUNK_SIZE = 512*1024
      56
      57  MAX_URI_LENGTH = 2048
      58
      59
      60 -class MediaUploadProgress(object):
      61    """Status of a resumable upload."""
      62
      63 -  def __init__(self, resumable_progress, total_size):
      64      """Constructor.
      65
      66      Args:
      67        resumable_progress: int, bytes sent so far.
      68        total_size: int, total bytes in complete upload, or None if the total
      69          upload size isn't known ahead of time.
      70      """
      71      self.resumable_progress = resumable_progress
      72      self.total_size = total_size
      73
      74 -  def progress(self):
      75      """Percent of upload completed, as a float.
      76
      77      Returns:
      78        the percentage complete as a float, returning 0.0 if the total size of
      79        the upload is unknown.
      80      """
      81      if self.total_size is not None:
      82        return float(self.resumable_progress) / float(self.total_size)
      83      else:
      84        return 0.0
      85
      86
      87 -class MediaDownloadProgress(object):
      88    """Status of a resumable download."""
      89
      90 -  def __init__(self, resumable_progress, total_size):
      91      """Constructor.
      92
      93      Args:
      94        resumable_progress: int, bytes received so far.
      95        total_size: int, total bytes in complete download.
      96      """
      97      self.resumable_progress = resumable_progress
      98      self.total_size = total_size
      99
     100 -  def progress(self):
     101      """Percent of download completed, as a float.
     102
     103      Returns:
     104        the percentage complete as a float, returning 0.0 if the total size of
     105        the download is unknown.
     106      """
     107      if self.total_size is not None:
     108        return float(self.resumable_progress) / float(self.total_size)
     109      else:
     110        return 0.0
     111
     112
     113 -class MediaUpload(object):
     114    """Describes a media object to upload.
     115
     116    Base class that defines the interface of MediaUpload subclasses.
     117
     118    Note that subclasses of MediaUpload may allow you to control the chunksize
     119    when uploading a media object. It is important to keep the size of the chunk
     120    as large as possible to keep the upload efficient. Other factors may influence
     121    the size of the chunk you use, particularly if you are working in an
     122    environment where individual HTTP requests may have a hardcoded time limit,
     123    such as under certain classes of requests under Google App Engine.
     124
     125    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     126    subclasses support using streams directly to upload data. Support for
     127    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     128    platform that stream will be used for uploading the media object. The support
     129    for streaming is indicated by has_stream() returning True. The stream() method
     130    should return an io.Base object that supports seek(). On platforms where the
     131    underlying httplib module supports streaming, for example Python 2.6 and
     132    later, the stream will be passed into the http library which will result in
     133    less memory being used and possibly faster uploads.
     134
     135    If you need to upload media that can't be uploaded using any of the existing
     136    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     137    needs.
     138    """
     139
     140 -  def chunksize(self):
     141      """Chunk size for resumable uploads.
     142
     143      Returns:
     144        Chunk size in bytes.
     145      """
     146      raise NotImplementedError()
     147
     148 -  def mimetype(self):
     149      """Mime type of the body.
     150
     151      Returns:
     152        Mime type.
     153      """
     154      return 'application/octet-stream'
     155
     156 -  def size(self):
     157      """Size of upload.
     158
     159      Returns:
     160        Size of the body, or None of the size is unknown.
     161      """
     162      return None
     163
     164 -  def resumable(self):
     165      """Whether this upload is resumable.
     166
     167      Returns:
     168        True if resumable upload or False.
     169      """
     170      return False
     171
     172 -  def getbytes(self, begin, end):
     173      """Get bytes from the media.
     174
     175      Args:
     176        begin: int, offset from beginning of file.
     177        length: int, number of bytes to read, starting at begin.
     178
     179      Returns:
     180        A string of bytes read. May be shorter than length if EOF was reached
     181        first.
     182      """
     183      raise NotImplementedError()
     184
     185 -  def has_stream(self):
     186      """Does the underlying upload support a streaming interface.
     187
     188      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     189      seekable() returns True.
     190
     191      Returns:
     192        True if the call to stream() will return an instance of a seekable io.Base
     193        subclass.
     194      """
     195      return False
     196
     197 -  def stream(self):
     198      """A stream interface to the data being uploaded.
     199
     200      Returns:
     201        The returned value is an io.IOBase subclass that supports seek, i.e.
     202        seekable() returns True.
     203      """
     204      raise NotImplementedError()
     205
     206    @util.positional(1)
     207 -  def _to_json(self, strip=None):
     208      """Utility function for creating a JSON representation of a MediaUpload.
     209
     210      Args:
     211        strip: array, An array of names of members to not include in the JSON.
     212
     213      Returns:
     214         string, a JSON representation of this instance, suitable to pass to
     215         from_json().
     216      """
     217      t = type(self)
     218      d = copy.copy(self.__dict__)
     219      if strip is not None:
     220        for member in strip:
     221          del d[member]
     222      d['_class'] = t.__name__
     223      d['_module'] = t.__module__
     224      return json.dumps(d)
     225
     226 -  def to_json(self):
     227      """Create a JSON representation of an instance of MediaUpload.
     228
     229      Returns:
     230         string, a JSON representation of this instance, suitable to pass to
     231         from_json().
     232      """
     233      return self._to_json()
     234
     235    @classmethod
     236 -  def new_from_json(cls, s):
     237      """Utility class method to instantiate a MediaUpload subclass from a JSON
     238      representation produced by to_json().
     239
     240      Args:
     241        s: string, JSON from to_json().
     242
     243      Returns:
     244        An instance of the subclass of MediaUpload that was serialized with
     245        to_json().
     246      """
     247      data = json.loads(s)
     248      # Find and call the right classmethod from_json() to restore the object.
     249      module = data['_module']
     250      m = __import__(module, fromlist=module.split('.')[:-1])
     251      kls = getattr(m, data['_class'])
     252      from_json = getattr(kls, 'from_json')
     253      return from_json(s)
     254
     255
     256 -class MediaIoBaseUpload(MediaUpload):
     257    """A MediaUpload for a io.Base objects.
     258
     259    Note that the Python file object is compatible with io.Base and can be used
     260    with this class also.
     261
     262      fh = io.BytesIO('...Some data to upload...')
     263      media = MediaIoBaseUpload(fh, mimetype='image/png',
     264        chunksize=1024*1024, resumable=True)
     265      farm.animals().insert(
     266          id='cow',
     267          name='cow.png',
     268          media_body=media).execute()
     269
     270    Depending on the platform you are working on, you may pass -1 as the
     271    chunksize, which indicates that the entire file should be uploaded in a single
     272    request. If the underlying platform supports streams, such as Python 2.6 or
     273    later, then this can be very efficient as it avoids multiple connections, and
     274    also avoids loading the entire file into memory before sending it. Note that
     275    Google App Engine has a 5MB limit on request size, so you should never set
     276    your chunksize larger than 5MB, or to -1.
     277    """
     278
     279    @util.positional(3)
     280 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     281        resumable=False):
     282      """Constructor.
     283
     284      Args:
     285        fd: io.Base or file object, The source of the bytes to upload. MUST be
     286          opened in blocking mode, do not use streams opened in non-blocking mode.
     287          The given stream must be seekable, that is, it must be able to call
     288          seek() on fd.
     289        mimetype: string, Mime-type of the file.
     290        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     291          used if resumable=True. Pass in a value of -1 if the file is to be
     292          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     293          on request size, so you should never set your chunksize larger than 5MB,
     294          or to -1.
     295        resumable: bool, True if this is a resumable upload. False means upload
     296          in a single request.
     297      """
     298      super(MediaIoBaseUpload, self).__init__()
     299      self._fd = fd
     300      self._mimetype = mimetype
     301      if not (chunksize == -1 or chunksize > 0):
     302        raise InvalidChunkSizeError()
     303      self._chunksize = chunksize
     304      self._resumable = resumable
     305
     306      self._fd.seek(0, os.SEEK_END)
     307      self._size = self._fd.tell()
     308
     309 -  def chunksize(self):
     310      """Chunk size for resumable uploads.
     311
     312      Returns:
     313        Chunk size in bytes.
     314      """
     315      return self._chunksize
     316
     317 -  def mimetype(self):
     318      """Mime type of the body.
     319
     320      Returns:
     321        Mime type.
     322      """
     323      return self._mimetype
     324
     325 -  def size(self):
     326      """Size of upload.
     327
     328      Returns:
     329        Size of the body, or None of the size is unknown.
     330      """
     331      return self._size
     332
     333 -  def resumable(self):
     334      """Whether this upload is resumable.
     335
     336      Returns:
     337        True if resumable upload or False.
     338      """
     339      return self._resumable
     340
     341 -  def getbytes(self, begin, length):
     342      """Get bytes from the media.
     343
     344      Args:
     345        begin: int, offset from beginning of file.
     346        length: int, number of bytes to read, starting at begin.
     347
     348      Returns:
     349        A string of bytes read. May be shorted than length if EOF was reached
     350        first.
     351      """
     352      self._fd.seek(begin)
     353      return self._fd.read(length)
     354
     355 -  def has_stream(self):
     356      """Does the underlying upload support a streaming interface.
     357
     358      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     359      seekable() returns True.
     360
     361      Returns:
     362        True if the call to stream() will return an instance of a seekable io.Base
     363        subclass.
     364      """
     365      return True
     366
     367 -  def stream(self):
     368      """A stream interface to the data being uploaded.
     369
     370      Returns:
     371        The returned value is an io.IOBase subclass that supports seek, i.e.
     372        seekable() returns True.
     373      """
     374      return self._fd
     375
     376 -  def to_json(self):
     377      """This upload type is not serializable."""
     378      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     379
     380
     381 -class MediaFileUpload(MediaIoBaseUpload):
     382    """A MediaUpload for a file.
     383
     384    Construct a MediaFileUpload and pass as the media_body parameter of the
     385    method. For example, if we had a service that allowed uploading images:
     386
     387
     388      media = MediaFileUpload('cow.png', mimetype='image/png',
     389        chunksize=1024*1024, resumable=True)
     390      farm.animals().insert(
     391          id='cow',
     392          name='cow.png',
     393          media_body=media).execute()
     394
     395    Depending on the platform you are working on, you may pass -1 as the
     396    chunksize, which indicates that the entire file should be uploaded in a single
     397    request. If the underlying platform supports streams, such as Python 2.6 or
     398    later, then this can be very efficient as it avoids multiple connections, and
     399    also avoids loading the entire file into memory before sending it. Note that
     400    Google App Engine has a 5MB limit on request size, so you should never set
     401    your chunksize larger than 5MB, or to -1.
     402    """
     403
     404    @util.positional(2)
     405 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     406                 resumable=False):
     407      """Constructor.
     408
     409      Args:
     410        filename: string, Name of the file.
     411        mimetype: string, Mime-type of the file. If None then a mime-type will be
     412          guessed from the file extension.
     413        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     414          used if resumable=True. Pass in a value of -1 if the file is to be
     415          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     416          on request size, so you should never set your chunksize larger than 5MB,
     417          or to -1.
     418        resumable: bool, True if this is a resumable upload. False means upload
     419          in a single request.
     420      """
     421      self._filename = filename
     422      fd = open(self._filename, 'rb')
     423      if mimetype is None:
     424        (mimetype, encoding) = mimetypes.guess_type(filename)
     425      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     426                                            resumable=resumable)
     427
     428 -  def to_json(self):
     429      """Creating a JSON representation of an instance of MediaFileUpload.
     430
     431      Returns:
     432         string, a JSON representation of this instance, suitable to pass to
     433         from_json().
     434      """
     435      return self._to_json(strip=['_fd'])
     436
     437    @staticmethod
     438 -  def from_json(s):
     439      d = json.loads(s)
     440      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     441                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     442
     443
     444 -class MediaInMemoryUpload(MediaIoBaseUpload):
     445    """MediaUpload for a chunk of bytes.
     446
     447    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     448    the stream.
     449    """
     450
     451    @util.positional(2)
     452 -  def __init__(self, body, mimetype='application/octet-stream',
     453                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     454      """Create a new MediaInMemoryUpload.
     455
     456    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     457    the stream.
     458
     459    Args:
     460      body: string, Bytes of body content.
     461      mimetype: string, Mime-type of the file or default of
     462        'application/octet-stream'.
     463      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     464        used if resumable=True.
     465      resumable: bool, True if this is a resumable upload. False means upload
     466        in a single request.
     467      """
     468      fd = StringIO.StringIO(body)
     469      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     470                                                resumable=resumable)
     471
     472
     473 -class MediaIoBaseDownload(object):
     474    """"Download media resources.
     475
     476    Note that the Python file object is compatible with io.Base and can be used
     477    with this class also.
     478
     479
     480    Example:
     481      request = farms.animals().get_media(id='cow')
     482      fh = io.FileIO('cow.png', mode='wb')
     483      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     484
     485      done = False
     486      while done is False:
     487        status, done = downloader.next_chunk()
     488        if status:
     489          print "Download %d%%." % int(status.progress() * 100)
     490      print "Download Complete!"
     491    """
     492
     493    @util.positional(3)
     494 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     495      """Constructor.
     496
     497      Args:
     498        fd: io.Base or file object, The stream in which to write the downloaded
     499          bytes.
     500        request: googleapiclient.http.HttpRequest, the media request to perform in
     501          chunks.
     502        chunksize: int, File will be downloaded in chunks of this many bytes.
     503      """
     504      self._fd = fd
     505      self._request = request
     506      self._uri = request.uri
     507      self._chunksize = chunksize
     508      self._progress = 0
     509      self._total_size = None
     510      self._done = False
     511
     512      # Stubs for testing.
     513      self._sleep = time.sleep
     514      self._rand = random.random
     515
     516    @util.positional(1)
     517 -  def next_chunk(self, num_retries=0):
     518      """Get the next chunk of the download.
     519
     520      Args:
     521        num_retries: Integer, number of times to retry 500's with randomized
     522              exponential backoff. If all retries fail, the raised HttpError
     523              represents the last request. If zero (default), we attempt the
     524              request only once.
     525
     526      Returns:
     527        (status, done): (MediaDownloadStatus, boolean)
     528           The value of 'done' will be True when the media has been fully
     529           downloaded.
     530
     531      Raises:
     532        googleapiclient.errors.HttpError if the response was not a 2xx.
     533        httplib2.HttpLib2Error if a transport error has occured.
     534      """
     535      headers = {
     536          'range': 'bytes=%d-%d' % (
     537              self._progress, self._progress + self._chunksize)
     538          }
     539      http = self._request.http
     540
     541      for retry_num in xrange(num_retries + 1):
     542        if retry_num > 0:
     543          self._sleep(self._rand() * 2**retry_num)
     544          logging.warning(
     545              'Retry #%d for media download: GET %s, following status: %d'
     546              % (retry_num, self._uri, resp.status))
     547
     548        resp, content = http.request(self._uri, headers=headers)
     549        if resp.status < 500:
     550          break
     551
     552      if resp.status in [200, 206]:
     553        if 'content-location' in resp and resp['content-location'] != self._uri:
     554          self._uri = resp['content-location']
     555        self._progress += len(content)
     556        self._fd.write(content)
     557
     558        if 'content-range' in resp:
     559          content_range = resp['content-range']
     560          length = content_range.rsplit('/', 1)[1]
     561          self._total_size = int(length)
     562
     563        if self._progress == self._total_size:
     564          self._done = True
     565        return MediaDownloadProgress(self._progress, self._total_size), self._done
     566      else:
     567        raise HttpError(resp, content, uri=self._uri)
     568
     569
     570 -class _StreamSlice(object):
     571    """Truncated stream.
     572
     573    Takes a stream and presents a stream that is a slice of the original stream.
     574    This is used when uploading media in chunks. In later versions of Python a
     575    stream can be passed to httplib in place of the string of data to send. The
     576    problem is that httplib just blindly reads to the end of the stream. This
     577    wrapper presents a virtual stream that only reads to the end of the chunk.
     578    """
     579
     580 -  def __init__(self, stream, begin, chunksize):
     581      """Constructor.
     582
     583      Args:
     584        stream: (io.Base, file object), the stream to wrap.
     585        begin: int, the seek position the chunk begins at.
     586        chunksize: int, the size of the chunk.
     587      """
     588      self._stream = stream
     589      self._begin = begin
     590      self._chunksize = chunksize
     591      self._stream.seek(begin)
     592
     593 -  def read(self, n=-1):
     594      """Read n bytes.
     595
     596      Args:
     597        n, int, the number of bytes to read.
     598
     599      Returns:
     600        A string of length 'n', or less if EOF is reached.
     601      """
     602      # The data left available to read sits in [cur, end)
     603      cur = self._stream.tell()
     604      end = self._begin + self._chunksize
     605      if n == -1 or cur + n > end:
     606        n = end - cur
     607      return self._stream.read(n)
     608
     609
     610 -class HttpRequest(object):
     611    """Encapsulates a single HTTP request."""
     612
     613    @util.positional(4)
     614 -  def __init__(self, http, postproc, uri,
     615                 method='GET',
     616                 body=None,
     617                 headers=None,
     618                 methodId=None,
     619                 resumable=None):
     620      """Constructor for an HttpRequest.
     621
     622      Args:
     623        http: httplib2.Http, the transport object to use to make a request
     624        postproc: callable, called on the HTTP response and content to transform
     625                  it into a data object before returning, or raising an exception
     626                  on an error.
     627        uri: string, the absolute URI to send the request to
     628        method: string, the HTTP method to use
     629        body: string, the request body of the HTTP request,
     630        headers: dict, the HTTP request headers
     631        methodId: string, a unique identifier for the API method being called.
     632        resumable: MediaUpload, None if this is not a resumbale request.
     633      """
     634      self.uri = uri
     635      self.method = method
     636      self.body = body
     637      self.headers = headers or {}
     638      self.methodId = methodId
     639      self.http = http
     640      self.postproc = postproc
     641      self.resumable = resumable
     642      self.response_callbacks = []
     643      self._in_error_state = False
     644
     645      # Pull the multipart boundary out of the content-type header.
     646      major, minor, params = mimeparse.parse_mime_type(
     647          headers.get('content-type', 'application/json'))
     648
     649      # The size of the non-media part of the request.
     650      self.body_size = len(self.body or '')
     651
     652      # The resumable URI to send chunks to.
     653      self.resumable_uri = None
     654
     655      # The bytes that have been uploaded.
     656      self.resumable_progress = 0
     657
     658      # Stubs for testing.
     659      self._rand = random.random
     660      self._sleep = time.sleep
     661
     662    @util.positional(1)
     663 -  def execute(self, http=None, num_retries=0):
     664      """Execute the request.
     665
     666      Args:
     667        http: httplib2.Http, an http object to be used in place of the
     668              one the HttpRequest request object was constructed with.
     669        num_retries: Integer, number of times to retry 500's with randomized
     670              exponential backoff. If all retries fail, the raised HttpError
     671              represents the last request. If zero (default), we attempt the
     672              request only once.
     673
     674      Returns:
     675        A deserialized object model of the response body as determined
     676        by the postproc.
     677
     678      Raises:
     679        googleapiclient.errors.HttpError if the response was not a 2xx.
     680        httplib2.HttpLib2Error if a transport error has occured.
     681      """
     682      if http is None:
     683        http = self.http
     684
     685      if self.resumable:
     686        body = None
     687        while body is None:
     688          _, body = self.next_chunk(http=http, num_retries=num_retries)
     689        return body
     690
     691      # Non-resumable case.
     692
     693      if 'content-length' not in self.headers:
     694        self.headers['content-length'] = str(self.body_size)
     695      # If the request URI is too long then turn it into a POST request.
     696      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     697        self.method = 'POST'
     698        self.headers['x-http-method-override'] = 'GET'
     699        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     700        parsed = urlparse.urlparse(self.uri)
     701        self.uri = urlparse.urlunparse(
     702            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     703             None)
     704            )
     705        self.body = parsed.query
     706        self.headers['content-length'] = str(len(self.body))
     707
     708      # Handle retries for server-side errors.
     709      for retry_num in xrange(num_retries + 1):
     710        if retry_num > 0:
     711          self._sleep(self._rand() * 2**retry_num)
     712          logging.warning('Retry #%d for request: %s %s, following status: %d'
     713                          % (retry_num, self.method, self.uri, resp.status))
     714
     715        resp, content = http.request(str(self.uri), method=str(self.method),
     716                                     body=self.body, headers=self.headers)
     717        if resp.status < 500:
     718          break
     719
     720      for callback in self.response_callbacks:
     721        callback(resp)
     722      if resp.status >= 300:
     723        raise HttpError(resp, content, uri=self.uri)
     724      return self.postproc(resp, content)
     725
     726    @util.positional(2)
     727 -  def add_response_callback(self, cb):
     728      """add_response_headers_callback
     729
     730      Args:
     731        cb: Callback to be called on receiving the response headers, of signature:
     732
     733        def cb(resp):
     734          # Where resp is an instance of httplib2.Response
     735      """
     736      self.response_callbacks.append(cb)
     737
     738    @util.positional(1)
     739 -  def next_chunk(self, http=None, num_retries=0):
     740      """Execute the next step of a resumable upload.
     741
     742      Can only be used if the method being executed supports media uploads and
     743      the MediaUpload object passed in was flagged as using resumable upload.
     744
     745      Example:
     746
     747        media = MediaFileUpload('cow.png', mimetype='image/png',
     748                                chunksize=1000, resumable=True)
     749        request = farm.animals().insert(
     750            id='cow',
     751            name='cow.png',
     752            media_body=media)
     753
     754        response = None
     755        while response is None:
     756          status, response = request.next_chunk()
     757          if status:
     758            print "Upload %d%% complete." % int(status.progress() * 100)
     759
     760
     761      Args:
     762        http: httplib2.Http, an http object to be used in place of the
     763              one the HttpRequest request object was constructed with.
     764        num_retries: Integer, number of times to retry 500's with randomized
     765              exponential backoff. If all retries fail, the raised HttpError
     766              represents the last request. If zero (default), we attempt the
     767              request only once.
     768
     769      Returns:
     770        (status, body): (ResumableMediaStatus, object)
     771           The body will be None until the resumable media is fully uploaded.
     772
     773      Raises:
     774        googleapiclient.errors.HttpError if the response was not a 2xx.
     775        httplib2.HttpLib2Error if a transport error has occured.
     776      """
     777      if http is None:
     778        http = self.http
     779
     780      if self.resumable.size() is None:
     781        size = '*'
     782      else:
     783        size = str(self.resumable.size())
     784
     785      if self.resumable_uri is None:
     786        start_headers = copy.copy(self.headers)
     787        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     788        if size != '*':
     789          start_headers['X-Upload-Content-Length'] = size
     790        start_headers['content-length'] = str(self.body_size)
     791
     792        for retry_num in xrange(num_retries + 1):
     793          if retry_num > 0:
     794            self._sleep(self._rand() * 2**retry_num)
     795            logging.warning(
     796                'Retry #%d for resumable URI request: %s %s, following status: %d'
     797                % (retry_num, self.method, self.uri, resp.status))
     798
     799          resp, content = http.request(self.uri, method=self.method,
     800                                       body=self.body,
     801                                       headers=start_headers)
     802          if resp.status < 500:
     803            break
     804
     805        if resp.status == 200 and 'location' in resp:
     806          self.resumable_uri = resp['location']
     807        else:
     808          raise ResumableUploadError(resp, content)
     809      elif self._in_error_state:
     810        # If we are in an error state then query the server for current state of
     811        # the upload by sending an empty PUT and reading the 'range' header in
     812        # the response.
     813        headers = {
     814            'Content-Range': 'bytes */%s' % size,
     815            'content-length': '0'
     816            }
     817        resp, content = http.request(self.resumable_uri, 'PUT',
     818                                     headers=headers)
     819        status, body = self._process_response(resp, content)
     820        if body:
     821          # The upload was complete.
     822          return (status, body)
     823
     824      # The httplib.request method can take streams for the body parameter, but
     825      # only in Python 2.6 or later. If a stream is available under those
     826      # conditions then use it as the body argument.
     827      if self.resumable.has_stream() and sys.version_info[1] >= 6:
     828        data = self.resumable.stream()
     829        if self.resumable.chunksize() == -1:
     830          data.seek(self.resumable_progress)
     831          chunk_end = self.resumable.size() - self.resumable_progress - 1
     832        else:
     833          # Doing chunking with a stream, so wrap a slice of the stream.
     834          data = _StreamSlice(data, self.resumable_progress,
     835                              self.resumable.chunksize())
     836          chunk_end = min(
     837              self.resumable_progress + self.resumable.chunksize() - 1,
     838              self.resumable.size() - 1)
     839      else:
     840        data = self.resumable.getbytes(
     841            self.resumable_progress, self.resumable.chunksize())
     842
     843        # A short read implies that we are at EOF, so finish the upload.
     844        if len(data) < self.resumable.chunksize():
     845          size = str(self.resumable_progress + len(data))
     846
     847        chunk_end = self.resumable_progress + len(data) - 1
     848
     849      headers = {
     850          'Content-Range': 'bytes %d-%d/%s' % (
     851              self.resumable_progress, chunk_end, size),
     852          # Must set the content-length header here because httplib can't
     853          # calculate the size when working with _StreamSlice.
     854          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     855          }
     856
     857      for retry_num in xrange(num_retries + 1):
     858        if retry_num > 0:
     859          self._sleep(self._rand() * 2**retry_num)
     860          logging.warning(
     861              'Retry #%d for media upload: %s %s, following status: %d'
     862              % (retry_num, self.method, self.uri, resp.status))
     863
     864        try:
     865          resp, content = http.request(self.resumable_uri, method='PUT',
     866                                       body=data,
     867                                       headers=headers)
     868        except:
     869          self._in_error_state = True
     870          raise
     871        if resp.status < 500:
     872          break
     873
     874      return self._process_response(resp, content)
     875
     876 -  def _process_response(self, resp, content):
     877      """Process the response from a single chunk upload.
     878
     879      Args:
     880        resp: httplib2.Response, the response object.
     881        content: string, the content of the response.
     882
     883      Returns:
     884        (status, body): (ResumableMediaStatus, object)
     885           The body will be None until the resumable media is fully uploaded.
     886
     887      Raises:
     888        googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
     889      """
     890      if resp.status in [200, 201]:
     891        self._in_error_state = False
     892        return None, self.postproc(resp, content)
     893      elif resp.status == 308:
     894        self._in_error_state = False
     895        # A "308 Resume Incomplete" indicates we are not done.
     896        self.resumable_progress = int(resp['range'].split('-')[1]) + 1
     897        if 'location' in resp:
     898          self.resumable_uri = resp['location']
     899      else:
     900        self._in_error_state = True
     901        raise HttpError(resp, content, uri=self.uri)
     902
     903      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
     904              None)
     905
     906 -  def to_json(self):
     907      """Returns a JSON representation of the HttpRequest."""
     908      d = copy.copy(self.__dict__)
     909      if d['resumable'] is not None:
     910        d['resumable'] = self.resumable.to_json()
     911      del d['http']
     912      del d['postproc']
     913      del d['_sleep']
     914      del d['_rand']
     915
     916      return json.dumps(d)
     917
     918    @staticmethod
     919 -  def from_json(s, http, postproc):
     920      """Returns an HttpRequest populated with info from a JSON object."""
     921      d = json.loads(s)
     922      if d['resumable'] is not None:
     923        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
     924      return HttpRequest(
     925          http,
     926          postproc,
     927          uri=d['uri'],
     928          method=d['method'],
     929          body=d['body'],
     930          headers=d['headers'],
     931          methodId=d['methodId'],
     932          resumable=d['resumable'])
     933
     934
     935 -class BatchHttpRequest(object):
     936    """Batches multiple HttpRequest objects into a single HTTP request.
     937
     938    Example:
     939      from googleapiclient.http import BatchHttpRequest
     940
     941      def list_animals(request_id, response, exception):
     942        \"\"\"Do something with the animals list response.\"\"\"
     943        if exception is not None:
     944          # Do something with the exception.
     945          pass
     946        else:
     947          # Do something with the response.
     948          pass
     949
     950      def list_farmers(request_id, response, exception):
     951        \"\"\"Do something with the farmers list response.\"\"\"
     952        if exception is not None:
     953          # Do something with the exception.
     954          pass
     955        else:
     956          # Do something with the response.
     957          pass
     958
     959      service = build('farm', 'v2')
     960
     961      batch = BatchHttpRequest()
     962
     963      batch.add(service.animals().list(), list_animals)
     964      batch.add(service.farmers().list(), list_farmers)
     965      batch.execute(http=http)
     966    """
     967
     968    @util.positional(1)
     969 -  def __init__(self, callback=None, batch_uri=None):
     970      """Constructor for a BatchHttpRequest.
     971
     972      Args:
     973        callback: callable, A callback to be called for each response, of the
     974          form callback(id, response, exception). The first parameter is the
     975          request id, and the second is the deserialized response object. The
     976          third is an googleapiclient.errors.HttpError exception object if an HTTP error
     977          occurred while processing the request, or None if no error occurred.
     978        batch_uri: string, URI to send batch requests to.
     979      """
     980      if batch_uri is None:
     981        batch_uri = 'https://www.googleapis.com/batch'
     982      self._batch_uri = batch_uri
     983
     984      # Global callback to be called for each individual response in the batch.
     985      self._callback = callback
     986
     987      # A map from id to request.
     988      self._requests = {}
     989
     990      # A map from id to callback.
     991      self._callbacks = {}
     992
     993      # List of request ids, in the order in which they were added.
     994      self._order = []
     995
     996      # The last auto generated id.
     997      self._last_auto_id = 0
     998
     999      # Unique ID on which to base the Content-ID headers.
    1000      self._base_id = None
    1001
    1002      # A map from request id to (httplib2.Response, content) response pairs
    1003      self._responses = {}
    1004
    1005      # A map of id(Credentials) that have been refreshed.
    1006      self._refreshed_credentials = {}
    1007
    1008 -  def _refresh_and_apply_credentials(self, request, http):
    1009      """Refresh the credentials and apply to the request.
    1010
    1011      Args:
    1012        request: HttpRequest, the request.
    1013        http: httplib2.Http, the global http object for the batch.
    1014      """
    1015      # For the credentials to refresh, but only once per refresh_token
    1016      # If there is no http per the request then refresh the http passed in
    1017      # via execute()
    1018      creds = None
    1019      if request.http is not None and hasattr(request.http.request,
    1020          'credentials'):
    1021        creds = request.http.request.credentials
    1022      elif http is not None and hasattr(http.request, 'credentials'):
    1023        creds = http.request.credentials
    1024      if creds is not None:
    1025        if id(creds) not in self._refreshed_credentials:
    1026          creds.refresh(http)
    1027          self._refreshed_credentials[id(creds)] = 1
    1028
    1029      # Only apply the credentials if we are using the http object passed in,
    1030      # otherwise apply() will get called during _serialize_request().
    1031      if request.http is None or not hasattr(request.http.request,
    1032          'credentials'):
    1033        creds.apply(request.headers)
    1034
    1035 -  def _id_to_header(self, id_):
    1036      """Convert an id to a Content-ID header value.
    1037
    1038      Args:
    1039        id_: string, identifier of individual request.
    1040
    1041      Returns:
    1042        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1043        the value because Content-ID headers are supposed to be universally
    1044        unique.
    1045      """
    1046      if self._base_id is None:
    1047        self._base_id = uuid.uuid4()
    1048
    1049      return '<%s+%s>' % (self._base_id, urllib.quote(id_))
    1050
    1051 -  def _header_to_id(self, header):
    1052      """Convert a Content-ID header value to an id.
    1053
    1054      Presumes the Content-ID header conforms to the format that _id_to_header()
    1055      returns.
    1056
    1057      Args:
    1058        header: string, Content-ID header value.
    1059
    1060      Returns:
    1061        The extracted id value.
    1062
    1063      Raises:
    1064        BatchError if the header is not in the expected format.
    1065      """
    1066      if header[0] != '<' or header[-1] != '>':
    1067        raise BatchError("Invalid value for Content-ID: %s" % header)
    1068      if '+' not in header:
    1069        raise BatchError("Invalid value for Content-ID: %s" % header)
    1070      base, id_ = header[1:-1].rsplit('+', 1)
    1071
    1072      return urllib.unquote(id_)
    1073
    1074 -  def _serialize_request(self, request):
    1075      """Convert an HttpRequest object into a string.
    1076
    1077      Args:
    1078        request: HttpRequest, the request to serialize.
    1079
    1080      Returns:
    1081        The request as a string in application/http format.
    1082      """
    1083      # Construct status line
    1084      parsed = urlparse.urlparse(request.uri)
    1085      request_line = urlparse.urlunparse(
    1086          (None, None, parsed.path, parsed.params, parsed.query, None)
    1087          )
    1088      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1089      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1090      msg = MIMENonMultipart(major, minor)
    1091      headers = request.headers.copy()
    1092
    1093      if request.http is not None and hasattr(request.http.request,
    1094          'credentials'):
    1095        request.http.request.credentials.apply(headers)
    1096
    1097      # MIMENonMultipart adds its own Content-Type header.
    1098      if 'content-type' in headers:
    1099        del headers['content-type']
    1100
    1101      for key, value in headers.iteritems():
    1102        msg[key] = value
    1103      msg['Host'] = parsed.netloc
    1104      msg.set_unixfrom(None)
    1105
    1106      if request.body is not None:
    1107        msg.set_payload(request.body)
    1108        msg['content-length'] = str(len(request.body))
    1109
    1110      # Serialize the mime message.
    1111      fp = StringIO.StringIO()
    1112      # maxheaderlen=0 means don't line wrap headers.
    1113      g = Generator(fp, maxheaderlen=0)
    1114      g.flatten(msg, unixfrom=False)
    1115      body = fp.getvalue()
    1116
    1117      # Strip off the \n\n that the MIME lib tacks onto the end of the payload.
    1118      if request.body is None:
    1119        body = body[:-2]
    1120
    1121      return status_line.encode('utf-8') + body
    1122
    1123 -  def _deserialize_response(self, payload):
    1124      """Convert string into httplib2 response and content.
    1125
    1126      Args:
    1127        payload: string, headers and body as a string.
    1128
    1129      Returns:
    1130        A pair (resp, content), such as would be returned from httplib2.request.
    1131      """
    1132      # Strip off the status line
    1133      status_line, payload = payload.split('\n', 1)
    1134      protocol, status, reason = status_line.split(' ', 2)
    1135
    1136      # Parse the rest of the response
    1137      parser = FeedParser()
    1138      parser.feed(payload)
    1139      msg = parser.close()
    1140      msg['status'] = status
    1141
    1142      # Create httplib2.Response from the parsed headers.
    1143      resp = httplib2.Response(msg)
    1144      resp.reason = reason
    1145      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1146
    1147      content = payload.split('\r\n\r\n', 1)[1]
    1148
    1149      return resp, content
    1150
    1151 -  def _new_id(self):
    1152      """Create a new id.
    1153
    1154      Auto incrementing number that avoids conflicts with ids already used.
    1155
    1156      Returns:
    1157         string, a new unique id.
    1158      """
    1159      self._last_auto_id += 1
    1160      while str(self._last_auto_id) in self._requests:
    1161        self._last_auto_id += 1
    1162      return str(self._last_auto_id)
    1163
    1164    @util.positional(2)
    1165 -  def add(self, request, callback=None, request_id=None):
    1166      """Add a new request.
    1167
    1168      Every callback added will be paired with a unique id, the request_id. That
    1169      unique id will be passed back to the callback when the response comes back
    1170      from the server. The default behavior is to have the library generate it's
    1171      own unique id. If the caller passes in a request_id then they must ensure
    1172      uniqueness for each request_id, and if they are not an exception is
    1173      raised. Callers should either supply all request_ids or nevery supply a
    1174      request id, to avoid such an error.
    1175
    1176      Args:
    1177        request: HttpRequest, Request to add to the batch.
    1178        callback: callable, A callback to be called for this response, of the
    1179          form callback(id, response, exception). The first parameter is the
    1180          request id, and the second is the deserialized response object. The
    1181          third is an googleapiclient.errors.HttpError exception object if an HTTP error
    1182          occurred while processing the request, or None if no errors occurred.
    1183        request_id: string, A unique id for the request. The id will be passed to
    1184          the callback with the response.
    1185
    1186      Returns:
    1187        None
    1188
    1189      Raises:
    1190        BatchError if a media request is added to a batch.
    1191        KeyError is the request_id is not unique.
    1192      """
    1193      if request_id is None:
    1194        request_id = self._new_id()
    1195      if request.resumable is not None:
    1196        raise BatchError("Media requests cannot be used in a batch request.")
    1197      if request_id in self._requests:
    1198        raise KeyError("A request with this ID already exists: %s" % request_id)
    1199      self._requests[request_id] = request
    1200      self._callbacks[request_id] = callback
    1201      self._order.append(request_id)
    1202
    1203 -  def _execute(self, http, order, requests):
    1204      """Serialize batch request, send to server, process response.
    1205
    1206      Args:
    1207        http: httplib2.Http, an http object to be used to make the request with.
    1208        order: list, list of request ids in the order they were added to the
    1209          batch.
    1210        request: list, list of request objects to send.
    1211
    1212      Raises:
    1213        httplib2.HttpLib2Error if a transport error has occured.
    1214        googleapiclient.errors.BatchError if the response is the wrong format.
    1215      """
    1216      message = MIMEMultipart('mixed')
    1217      # Message should not write out it's own headers.
    1218      setattr(message, '_write_headers', lambda self: None)
    1219
    1220      # Add all the individual requests.
    1221      for request_id in order:
    1222        request = requests[request_id]
    1223
    1224        msg = MIMENonMultipart('application', 'http')
    1225        msg['Content-Transfer-Encoding'] = 'binary'
    1226        msg['Content-ID'] = self._id_to_header(request_id)
    1227
    1228        body = self._serialize_request(request)
    1229        msg.set_payload(body)
    1230        message.attach(msg)
    1231
    1232      # encode the body: note that we can't use `as_string`, because
    1233      # it plays games with `From ` lines.
    1234      fp = StringIO.StringIO()
    1235      g = Generator(fp, mangle_from_=False)
    1236      g.flatten(message, unixfrom=False)
    1237      body = fp.getvalue()
    1238
    1239      headers = {}
    1240      headers['content-type'] = ('multipart/mixed; '
    1241                                 'boundary="%s"') % message.get_boundary()
    1242
    1243      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1244                                   headers=headers)
    1245
    1246      if resp.status >= 300:
    1247        raise HttpError(resp, content, uri=self._batch_uri)
    1248
    1249      # Now break out the individual responses and store each one.
    1250      boundary, _ = content.split(None, 1)
    1251
    1252      # Prepend with a content-type header so FeedParser can handle it.
    1253      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1254      for_parser = header + content
    1255
    1256      parser = FeedParser()
    1257      parser.feed(for_parser)
    1258      mime_response = parser.close()
    1259
    1260      if not mime_response.is_multipart():
    1261        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1262                         content=content)
    1263
    1264      for part in mime_response.get_payload():
    1265        request_id = self._header_to_id(part['Content-ID'])
    1266        response, content = self._deserialize_response(part.get_payload())
    1267        self._responses[request_id] = (response, content)
    1268
    1269    @util.positional(1)
    1270 -  def execute(self, http=None):
    1271      """Execute all the requests as a single batched HTTP request.
    1272
    1273      Args:
    1274        http: httplib2.Http, an http object to be used in place of the one the
    1275          HttpRequest request object was constructed with. If one isn't supplied
    1276          then use a http object from the requests in this batch.
    1277
    1278      Returns:
    1279        None
    1280
    1281      Raises:
    1282        httplib2.HttpLib2Error if a transport error has occured.
    1283        googleapiclient.errors.BatchError if the response is the wrong format.
    1284      """
    1285
    1286      # If http is not supplied use the first valid one given in the requests.
    1287      if http is None:
    1288        for request_id in self._order:
    1289          request = self._requests[request_id]
    1290          if request is not None:
    1291            http = request.http
    1292            break
    1293
    1294      if http is None:
    1295        raise ValueError("Missing a valid http object.")
    1296
    1297      self._execute(http, self._order, self._requests)
    1298
    1299      # Loop over all the requests and check for 401s. For each 401 request the
    1300      # credentials should be refreshed and then sent again in a separate batch.
    1301      redo_requests = {}
    1302      redo_order = []
    1303
    1304      for request_id in self._order:
    1305        resp, content = self._responses[request_id]
    1306        if resp['status'] == '401':
    1307          redo_order.append(request_id)
    1308          request = self._requests[request_id]
    1309          self._refresh_and_apply_credentials(request, http)
    1310          redo_requests[request_id] = request
    1311
    1312      if redo_requests:
    1313        self._execute(http, redo_order, redo_requests)
    1314
    1315      # Now process all callbacks that are erroring, and raise an exception for
    1316      # ones that return a non-2xx response? Or add extra parameter to callback
    1317      # that contains an HttpError?
    1318
    1319      for request_id in self._order:
    1320        resp, content = self._responses[request_id]
    1321
    1322        request = self._requests[request_id]
    1323        callback = self._callbacks[request_id]
    1324
    1325        response = None
    1326        exception = None
    1327        try:
    1328          if resp.status >= 300:
    1329            raise HttpError(resp, content, uri=request.uri)
    1330          response = request.postproc(resp, content)
    1331        except HttpError, e:
    1332          exception = e
    1333
    1334        if callback is not None:
    1335          callback(request_id, response, exception)
    1336        if self._callback is not None:
    1337          self._callback(request_id, response, exception)
    1338
    1339
    1340 -class HttpRequestMock(object):
    1341    """Mock of HttpRequest.
    1342
    1343    Do not construct directly, instead use RequestMockBuilder.
    1344    """
    1345
    1346 -  def __init__(self, resp, content, postproc):
    1347      """Constructor for HttpRequestMock
    1348
    1349      Args:
    1350        resp: httplib2.Response, the response to emulate coming from the request
    1351        content: string, the response body
    1352        postproc: callable, the post processing function usually supplied by
    1353                  the model class. See model.JsonModel.response() as an example.
    1354      """
    1355      self.resp = resp
    1356      self.content = content
    1357      self.postproc = postproc
    1358      if resp is None:
    1359        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1360      if 'reason' in self.resp:
    1361        self.resp.reason = self.resp['reason']
    1362
    1363 -  def execute(self, http=None):
    1364      """Execute the request.
    1365
    1366      Same behavior as HttpRequest.execute(), but the response is
    1367      mocked and not really from an HTTP request/response.
    1368      """
    1369      return self.postproc(self.resp, self.content)
    1370
    1371
    1372 -class RequestMockBuilder(object):
    1373    """A simple mock of HttpRequest
    1374
    1375      Pass in a dictionary to the constructor that maps request methodIds to
    1376      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1377      returned when that method is called. None may also be passed in for the
    1378      httplib2.Response, in which case a 200 OK response will be generated.
    1379      If an opt_expected_body (str or dict) is provided, it will be compared to
    1380      the body and UnexpectedBodyError will be raised on inequality.
    1381
    1382      Example:
    1383        response = '{"data": {"id": "tag:google.c...'
    1384        requestBuilder = RequestMockBuilder(
    1385          {
    1386            'plus.activities.get': (None, response),
    1387          }
    1388        )
    1389        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1390
    1391      Methods that you do not supply a response for will return a
    1392      200 OK with an empty string as the response content or raise an excpetion
    1393      if check_unexpected is set to True. The methodId is taken from the rpcName
    1394      in the discovery document.
    1395
    1396      For more details see the project wiki.
    1397    """
    1398
    1399 -  def __init__(self, responses, check_unexpected=False):
    1400      """Constructor for RequestMockBuilder
    1401
    1402      The constructed object should be a callable object
    1403      that can replace the class HttpResponse.
    1404
    1405      responses - A dictionary that maps methodIds into tuples
    1406                  of (httplib2.Response, content). The methodId
    1407                  comes from the 'rpcName' field in the discovery
    1408                  document.
    1409      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1410                         should be raised on unsupplied method.
    1411      """
    1412      self.responses = responses
    1413      self.check_unexpected = check_unexpected
    1414
    1415 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1416                 headers=None, methodId=None, resumable=None):
    1417      """Implements the callable interface that discovery.build() expects
    1418      of requestBuilder, which is to build an object compatible with
    1419      HttpRequest.execute(). See that method for the description of the
    1420      parameters and the expected response.
    1421      """
    1422      if methodId in self.responses:
    1423        response = self.responses[methodId]
    1424        resp, content = response[:2]
    1425        if len(response) > 2:
    1426          # Test the body against the supplied expected_body.
    1427          expected_body = response[2]
    1428          if bool(expected_body) != bool(body):
    1429            # Not expecting a body and provided one
    1430            # or expecting a body and not provided one.
    1431            raise UnexpectedBodyError(expected_body, body)
    1432          if isinstance(expected_body, str):
    1433            expected_body = json.loads(expected_body)
    1434          body = json.loads(body)
    1435          if body != expected_body:
    1436            raise UnexpectedBodyError(expected_body, body)
    1437        return HttpRequestMock(resp, content, postproc)
    1438      elif self.check_unexpected:
    1439        raise UnexpectedMethodError(methodId=methodId)
    1440      else:
    1441        model = JsonModel(False)
    1442        return HttpRequestMock(None, '{}', model.response)
    1443
    1444
    1445 -class HttpMock(object):
    1446    """Mock of httplib2.Http"""
    1447
    1448 -  def __init__(self, filename=None, headers=None):
    1449      """
    1450      Args:
    1451        filename: string, absolute filename to read response from
    1452        headers: dict, header to return with response
    1453      """
    1454      if headers is None:
    1455        headers = {'status': '200 OK'}
    1456      if filename:
    1457        f = file(filename, 'r')
    1458        self.data = f.read()
    1459        f.close()
    1460      else:
    1461        self.data = None
    1462      self.response_headers = headers
    1463      self.headers = None
    1464      self.uri = None
    1465      self.method = None
    1466      self.body = None
    1467      self.headers = None
    1468
    1469
    1470 -  def request(self, uri,
    1471                method='GET',
    1472                body=None,
    1473                headers=None,
    1474                redirections=1,
    1475                connection_type=None):
    1476      self.uri = uri
    1477      self.method = method
    1478      self.body = body
    1479      self.headers = headers
    1480      return httplib2.Response(self.response_headers), self.data
    1481
    1482
    1483 -class HttpMockSequence(object):
    1484    """Mock of httplib2.Http
    1485
    1486    Mocks a sequence of calls to request returning different responses for each
    1487    call. Create an instance initialized with the desired response headers
    1488    and content and then use as if an httplib2.Http instance.
    1489
    1490      http = HttpMockSequence([
    1491        ({'status': '401'}, ''),
    1492        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1493        ({'status': '200'}, 'echo_request_headers'),
    1494        ])
    1495      resp, content = http.request("http://examples.com")
    1496
    1497    There are special values you can pass in for content to trigger
    1498    behavours that are helpful in testing.
    1499
    1500    'echo_request_headers' means return the request headers in the response body
    1501    'echo_request_headers_as_json' means return the request headers in
    1502       the response body
    1503    'echo_request_body' means return the request body in the response body
    1504    'echo_request_uri' means return the request uri in the response body
    1505    """
    1506
    1507 -  def __init__(self, iterable):
    1508      """
    1509      Args:
    1510        iterable: iterable, a sequence of pairs of (headers, body)
    1511      """
    1512      self._iterable = iterable
    1513      self.follow_redirects = True
    1514
    1515 -  def request(self, uri,
    1516                method='GET',
    1517                body=None,
    1518                headers=None,
    1519                redirections=1,
    1520                connection_type=None):
    1521      resp, content = self._iterable.pop(0)
    1522      if content == 'echo_request_headers':
    1523        content = headers
    1524      elif content == 'echo_request_headers_as_json':
    1525        content = json.dumps(headers)
    1526      elif content == 'echo_request_body':
    1527        if hasattr(body, 'read'):
    1528          content = body.read()
    1529        else:
    1530          content = body
    1531      elif content == 'echo_request_uri':
    1532        content = uri
    1533      return httplib2.Response(resp), content
    1534
    1535
    1536 -def set_user_agent(http, user_agent):
    1537    """Set the user-agent on every request.
    1538
    1539    Args:
    1540       http - An instance of httplib2.Http
    1541           or something that acts like it.
    1542       user_agent: string, the value for the user-agent header.
    1543
    1544    Returns:
    1545       A modified instance of http that was passed in.
    1546
    1547    Example:
    1548
    1549      h = httplib2.Http()
    1550      h = set_user_agent(h, "my-app-name/6.0")
    1551
    1552    Most of the time the user-agent will be set doing auth, this is for the rare
    1553    cases where you are accessing an unauthenticated endpoint.
    1554    """
    1555    request_orig = http.request
    1556
    1557    # The closure that will replace 'httplib2.Http.request'.
    1558    def new_request(uri, method='GET', body=None, headers=None,
    1559                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1560                    connection_type=None):
    1561      """Modify the request headers to add the user-agent."""
    1562      if headers is None:
    1563        headers = {}
    1564      if 'user-agent' in headers:
    1565        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1566      else:
    1567        headers['user-agent'] = user_agent
    1568      resp, content = request_orig(uri, method, body, headers,
    1569                          redirections, connection_type)
    1570      return resp, content
    1571
    1572    http.request = new_request
    1573    return http
    1574
    1575
    1576 -def tunnel_patch(http):
    1577    """Tunnel PATCH requests over POST.
    1578    Args:
    1579       http - An instance of httplib2.Http
    1580           or something that acts like it.
    1581
    1582    Returns:
    1583       A modified instance of http that was passed in.
    1584
    1585    Example:
    1586
    1587      h = httplib2.Http()
    1588      h = tunnel_patch(h, "my-app-name/6.0")
    1589
    1590    Useful if you are running on a platform that doesn't support PATCH.
    1591    Apply this last if you are using OAuth 1.0, as changing the method
    1592    will result in a different signature.
    1593    """
    1594    request_orig = http.request
    1595
    1596    # The closure that will replace 'httplib2.Http.request'.
    1597    def new_request(uri, method='GET', body=None, headers=None,
    1598                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1599                    connection_type=None):
    1600      """Modify the request headers to add the user-agent."""
    1601      if headers is None:
    1602        headers = {}
    1603      if method == 'PATCH':
    1604        if 'oauth_token' in headers.get('authorization', ''):
    1605          logging.warning(
    1606              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1607        headers['x-http-method-override'] = "PATCH"
    1608        method = 'POST'
    1609      resp, content = request_orig(uri, method, body, headers,
    1610                          redirections, connection_type)
    1611      return resp, content
    1612
    1613    http.request = new_request
    1614    return http
    1615