1   
   2   
   3   
   4   
   5   
   6   
   7   
   8   
   9   
  10   
  11   
  12   
  13   
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actual HTTP request. 
  20  """ 
  21  from __future__ import absolute_import 
  22  import six 
  23  from six.moves import http_client 
  24  from six.moves import range 
  25   
  26  __author__ = "jcgregorio@google.com (Joe Gregorio)" 
  27   
  28  from six import BytesIO, StringIO 
  29  from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 
  30   
  31  import base64 
  32  import copy 
  33  import gzip 
  34  import httplib2 
  35  import json 
  36  import logging 
  37  import mimetypes 
  38  import os 
  39  import random 
  40  import socket 
  41  import sys 
  42  import time 
  43  import uuid 
  44   
  45   
  46  try: 
  47      import ssl 
  48  except ImportError: 
  49      _ssl_SSLError = object() 
  50  else: 
  51      _ssl_SSLError = ssl.SSLError 
  52   
  53  from email.generator import Generator 
  54  from email.mime.multipart import MIMEMultipart 
  55  from email.mime.nonmultipart import MIMENonMultipart 
  56  from email.parser import FeedParser 
  57   
  58  from googleapiclient import _helpers as util 
  59   
  60  from googleapiclient import _auth 
  61  from googleapiclient.errors import BatchError 
  62  from googleapiclient.errors import HttpError 
  63  from googleapiclient.errors import InvalidChunkSizeError 
  64  from googleapiclient.errors import ResumableUploadError 
  65  from googleapiclient.errors import UnexpectedBodyError 
  66  from googleapiclient.errors import UnexpectedMethodError 
  67  from googleapiclient.model import JsonModel 
  68   
  69   
  70  LOGGER = logging.getLogger(__name__) 
  71   
  72  DEFAULT_CHUNK_SIZE = 100 * 1024 * 1024 
  73   
  74  MAX_URI_LENGTH = 2048 
  75   
  76  MAX_BATCH_LIMIT = 1000 
  77   
  78  _TOO_MANY_REQUESTS = 429 
  79   
  80  DEFAULT_HTTP_TIMEOUT_SEC = 60 
  81   
  82  _LEGACY_BATCH_URI = "https://www.googleapis.com/batch" 
  83   
  84  if six.PY2: 
  85       
  86       
  87      ConnectionError = None 
  91      """Determines whether a response should be retried. 
  92   
  93    Args: 
  94      resp_status: The response status received. 
  95      content: The response content body. 
  96   
  97    Returns: 
  98      True if the response should be retried, otherwise False. 
  99    """ 
 100       
 101      if resp_status >= 500: 
 102          return True 
 103   
 104       
 105      if resp_status == _TOO_MANY_REQUESTS: 
 106          return True 
 107   
 108       
 109       
 110      if resp_status == six.moves.http_client.FORBIDDEN: 
 111           
 112          if not content: 
 113              return False 
 114   
 115           
 116          try: 
 117              data = json.loads(content.decode("utf-8")) 
 118              if isinstance(data, dict): 
 119                  reason = data["error"]["errors"][0]["reason"] 
 120              else: 
 121                  reason = data[0]["error"]["errors"]["reason"] 
 122          except (UnicodeDecodeError, ValueError, KeyError): 
 123              LOGGER.warning("Invalid JSON content from response: %s", content) 
 124              return False 
 125   
 126          LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 
 127   
 128           
 129          if reason in ("userRateLimitExceeded", "rateLimitExceeded"): 
 130              return True 
 131   
 132       
 133      return False 
  134   
 135   
 136 -def _retry_request( 
 137      http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs 
 138  ): 
  139      """Retries an HTTP request multiple times while handling errors. 
 140   
 141    If after all retries the request still fails, last error is either returned as 
 142    return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 
 143   
 144    Args: 
 145      http: Http object to be used to execute request. 
 146      num_retries: Maximum number of retries. 
 147      req_type: Type of the request (used for logging retries). 
 148      sleep, rand: Functions to sleep for random time between retries. 
 149      uri: URI to be requested. 
 150      method: HTTP method to be used. 
 151      args, kwargs: Additional arguments passed to http.request. 
 152   
 153    Returns: 
 154      resp, content - Response from the http request (may be HTTP 5xx). 
 155    """ 
 156      resp = None 
 157      content = None 
 158      exception = None 
 159      for retry_num in range(num_retries + 1): 
 160          if retry_num > 0: 
 161               
 162              sleep_time = rand() * 2 ** retry_num 
 163              LOGGER.warning( 
 164                  "Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s", 
 165                  sleep_time, 
 166                  retry_num, 
 167                  num_retries, 
 168                  req_type, 
 169                  method, 
 170                  uri, 
 171                  resp.status if resp else exception, 
 172              ) 
 173              sleep(sleep_time) 
 174   
 175          try: 
 176              exception = None 
 177              resp, content = http.request(uri, method, *args, **kwargs) 
 178           
 179          except _ssl_SSLError as ssl_error: 
 180              exception = ssl_error 
 181          except socket.timeout as socket_timeout: 
 182               
 183               
 184              exception = socket_timeout 
 185          except ConnectionError as connection_error: 
 186               
 187               
 188              exception = connection_error 
 189          except socket.error as socket_error: 
 190               
 191              if socket.errno.errorcode.get(socket_error.errno) not in { 
 192                  "WSAETIMEDOUT", 
 193                  "ETIMEDOUT", 
 194                  "EPIPE", 
 195                  "ECONNABORTED", 
 196              }: 
 197                  raise 
 198              exception = socket_error 
 199          except httplib2.ServerNotFoundError as server_not_found_error: 
 200              exception = server_not_found_error 
 201   
 202          if exception: 
 203              if retry_num == num_retries: 
 204                  raise exception 
 205              else: 
 206                  continue 
 207   
 208          if not _should_retry_response(resp.status, content): 
 209              break 
 210   
 211      return resp, content 
  212   
 239   
 265   
 408   
 532   
 607   
 642   
 750   
 753      """Truncated stream. 
 754   
 755    Takes a stream and presents a stream that is a slice of the original stream. 
 756    This is used when uploading media in chunks. In later versions of Python a 
 757    stream can be passed to httplib in place of the string of data to send. The 
 758    problem is that httplib just blindly reads to the end of the stream. This 
 759    wrapper presents a virtual stream that only reads to the end of the chunk. 
 760    """ 
 761   
 762 -    def __init__(self, stream, begin, chunksize): 
  763          """Constructor. 
 764   
 765      Args: 
 766        stream: (io.Base, file object), the stream to wrap. 
 767        begin: int, the seek position the chunk begins at. 
 768        chunksize: int, the size of the chunk. 
 769      """ 
 770          self._stream = stream 
 771          self._begin = begin 
 772          self._chunksize = chunksize 
 773          self._stream.seek(begin) 
  774   
 775 -    def read(self, n=-1): 
  776          """Read n bytes. 
 777   
 778      Args: 
 779        n, int, the number of bytes to read. 
 780   
 781      Returns: 
 782        A string of length 'n', or less if EOF is reached. 
 783      """ 
 784           
 785          cur = self._stream.tell() 
 786          end = self._begin + self._chunksize 
 787          if n == -1 or cur + n > end: 
 788              n = end - cur 
 789          return self._stream.read(n) 
   790   
 793      """Encapsulates a single HTTP request.""" 
 794   
 795      @util.positional(4) 
 796 -    def __init__( 
 797          self, 
 798          http, 
 799          postproc, 
 800          uri, 
 801          method="GET", 
 802          body=None, 
 803          headers=None, 
 804          methodId=None, 
 805          resumable=None, 
 806      ): 
  807          """Constructor for an HttpRequest. 
 808   
 809      Args: 
 810        http: httplib2.Http, the transport object to use to make a request 
 811        postproc: callable, called on the HTTP response and content to transform 
 812                  it into a data object before returning, or raising an exception 
 813                  on an error. 
 814        uri: string, the absolute URI to send the request to 
 815        method: string, the HTTP method to use 
 816        body: string, the request body of the HTTP request, 
 817        headers: dict, the HTTP request headers 
 818        methodId: string, a unique identifier for the API method being called. 
 819        resumable: MediaUpload, None if this is not a resumbale request. 
 820      """ 
 821          self.uri = uri 
 822          self.method = method 
 823          self.body = body 
 824          self.headers = headers or {} 
 825          self.methodId = methodId 
 826          self.http = http 
 827          self.postproc = postproc 
 828          self.resumable = resumable 
 829          self.response_callbacks = [] 
 830          self._in_error_state = False 
 831   
 832           
 833          self.body_size = len(self.body or "") 
 834   
 835           
 836          self.resumable_uri = None 
 837   
 838           
 839          self.resumable_progress = 0 
 840   
 841           
 842          self._rand = random.random 
 843          self._sleep = time.sleep 
  844   
 845      @util.positional(1) 
 846 -    def execute(self, http=None, num_retries=0): 
  847          """Execute the request. 
 848   
 849      Args: 
 850        http: httplib2.Http, an http object to be used in place of the 
 851              one the HttpRequest request object was constructed with. 
 852        num_retries: Integer, number of times to retry with randomized 
 853              exponential backoff. If all retries fail, the raised HttpError 
 854              represents the last request. If zero (default), we attempt the 
 855              request only once. 
 856   
 857      Returns: 
 858        A deserialized object model of the response body as determined 
 859        by the postproc. 
 860   
 861      Raises: 
 862        googleapiclient.errors.HttpError if the response was not a 2xx. 
 863        httplib2.HttpLib2Error if a transport error has occurred. 
 864      """ 
 865          if http is None: 
 866              http = self.http 
 867   
 868          if self.resumable: 
 869              body = None 
 870              while body is None: 
 871                  _, body = self.next_chunk(http=http, num_retries=num_retries) 
 872              return body 
 873   
 874           
 875   
 876          if "content-length" not in self.headers: 
 877              self.headers["content-length"] = str(self.body_size) 
 878           
 879           
 880          if len(self.uri) > MAX_URI_LENGTH and self.method == "GET": 
 881              self.method = "POST" 
 882              self.headers["x-http-method-override"] = "GET" 
 883              self.headers["content-type"] = "application/x-www-form-urlencoded" 
 884              parsed = urlparse(self.uri) 
 885              self.uri = urlunparse( 
 886                  (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, None) 
 887              ) 
 888              self.body = parsed.query 
 889              self.headers["content-length"] = str(len(self.body)) 
 890   
 891           
 892          resp, content = _retry_request( 
 893              http, 
 894              num_retries, 
 895              "request", 
 896              self._sleep, 
 897              self._rand, 
 898              str(self.uri), 
 899              method=str(self.method), 
 900              body=self.body, 
 901              headers=self.headers, 
 902          ) 
 903   
 904          for callback in self.response_callbacks: 
 905              callback(resp) 
 906          if resp.status >= 300: 
 907              raise HttpError(resp, content, uri=self.uri) 
 908          return self.postproc(resp, content) 
  909   
 910      @util.positional(2) 
 912          """add_response_headers_callback 
 913   
 914      Args: 
 915        cb: Callback to be called on receiving the response headers, of signature: 
 916   
 917        def cb(resp): 
 918          # Where resp is an instance of httplib2.Response 
 919      """ 
 920          self.response_callbacks.append(cb) 
  921   
 922      @util.positional(1) 
 924          """Execute the next step of a resumable upload. 
 925   
 926      Can only be used if the method being executed supports media uploads and 
 927      the MediaUpload object passed in was flagged as using resumable upload. 
 928   
 929      Example: 
 930   
 931        media = MediaFileUpload('cow.png', mimetype='image/png', 
 932                                chunksize=1000, resumable=True) 
 933        request = farm.animals().insert( 
 934            id='cow', 
 935            name='cow.png', 
 936            media_body=media) 
 937   
 938        response = None 
 939        while response is None: 
 940          status, response = request.next_chunk() 
 941          if status: 
 942            print "Upload %d%% complete." % int(status.progress() * 100) 
 943   
 944   
 945      Args: 
 946        http: httplib2.Http, an http object to be used in place of the 
 947              one the HttpRequest request object was constructed with. 
 948        num_retries: Integer, number of times to retry with randomized 
 949              exponential backoff. If all retries fail, the raised HttpError 
 950              represents the last request. If zero (default), we attempt the 
 951              request only once. 
 952   
 953      Returns: 
 954        (status, body): (ResumableMediaStatus, object) 
 955           The body will be None until the resumable media is fully uploaded. 
 956   
 957      Raises: 
 958        googleapiclient.errors.HttpError if the response was not a 2xx. 
 959        httplib2.HttpLib2Error if a transport error has occurred. 
 960      """ 
 961          if http is None: 
 962              http = self.http 
 963   
 964          if self.resumable.size() is None: 
 965              size = "*" 
 966          else: 
 967              size = str(self.resumable.size()) 
 968   
 969          if self.resumable_uri is None: 
 970              start_headers = copy.copy(self.headers) 
 971              start_headers["X-Upload-Content-Type"] = self.resumable.mimetype() 
 972              if size != "*": 
 973                  start_headers["X-Upload-Content-Length"] = size 
 974              start_headers["content-length"] = str(self.body_size) 
 975   
 976              resp, content = _retry_request( 
 977                  http, 
 978                  num_retries, 
 979                  "resumable URI request", 
 980                  self._sleep, 
 981                  self._rand, 
 982                  self.uri, 
 983                  method=self.method, 
 984                  body=self.body, 
 985                  headers=start_headers, 
 986              ) 
 987   
 988              if resp.status == 200 and "location" in resp: 
 989                  self.resumable_uri = resp["location"] 
 990              else: 
 991                  raise ResumableUploadError(resp, content) 
 992          elif self._in_error_state: 
 993               
 994               
 995               
 996              headers = {"Content-Range": "bytes */%s" % size, "content-length": "0"} 
 997              resp, content = http.request(self.resumable_uri, "PUT", headers=headers) 
 998              status, body = self._process_response(resp, content) 
 999              if body: 
1000                   
1001                  return (status, body) 
1002   
1003          if self.resumable.has_stream(): 
1004              data = self.resumable.stream() 
1005              if self.resumable.chunksize() == -1: 
1006                  data.seek(self.resumable_progress) 
1007                  chunk_end = self.resumable.size() - self.resumable_progress - 1 
1008              else: 
1009                   
1010                  data = _StreamSlice( 
1011                      data, self.resumable_progress, self.resumable.chunksize() 
1012                  ) 
1013                  chunk_end = min( 
1014                      self.resumable_progress + self.resumable.chunksize() - 1, 
1015                      self.resumable.size() - 1, 
1016                  ) 
1017          else: 
1018              data = self.resumable.getbytes( 
1019                  self.resumable_progress, self.resumable.chunksize() 
1020              ) 
1021   
1022               
1023              if len(data) < self.resumable.chunksize(): 
1024                  size = str(self.resumable_progress + len(data)) 
1025   
1026              chunk_end = self.resumable_progress + len(data) - 1 
1027   
1028          headers = { 
1029              "Content-Range": "bytes %d-%d/%s" 
1030              % (self.resumable_progress, chunk_end, size), 
1031               
1032               
1033              "Content-Length": str(chunk_end - self.resumable_progress + 1), 
1034          } 
1035   
1036          for retry_num in range(num_retries + 1): 
1037              if retry_num > 0: 
1038                  self._sleep(self._rand() * 2 ** retry_num) 
1039                  LOGGER.warning( 
1040                      "Retry #%d for media upload: %s %s, following status: %d" 
1041                      % (retry_num, self.method, self.uri, resp.status) 
1042                  ) 
1043   
1044              try: 
1045                  resp, content = http.request( 
1046                      self.resumable_uri, method="PUT", body=data, headers=headers 
1047                  ) 
1048              except: 
1049                  self._in_error_state = True 
1050                  raise 
1051              if not _should_retry_response(resp.status, content): 
1052                  break 
1053   
1054          return self._process_response(resp, content) 
 1055   
1057          """Process the response from a single chunk upload. 
1058   
1059      Args: 
1060        resp: httplib2.Response, the response object. 
1061        content: string, the content of the response. 
1062   
1063      Returns: 
1064        (status, body): (ResumableMediaStatus, object) 
1065           The body will be None until the resumable media is fully uploaded. 
1066   
1067      Raises: 
1068        googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 
1069      """ 
1070          if resp.status in [200, 201]: 
1071              self._in_error_state = False 
1072              return None, self.postproc(resp, content) 
1073          elif resp.status == 308: 
1074              self._in_error_state = False 
1075               
1076              try: 
1077                  self.resumable_progress = int(resp["range"].split("-")[1]) + 1 
1078              except KeyError: 
1079                   
1080                  self.resumable_progress = 0 
1081              if "location" in resp: 
1082                  self.resumable_uri = resp["location"] 
1083          else: 
1084              self._in_error_state = True 
1085              raise HttpError(resp, content, uri=self.uri) 
1086   
1087          return ( 
1088              MediaUploadProgress(self.resumable_progress, self.resumable.size()), 
1089              None, 
1090          ) 
 1091   
1093          """Returns a JSON representation of the HttpRequest.""" 
1094          d = copy.copy(self.__dict__) 
1095          if d["resumable"] is not None: 
1096              d["resumable"] = self.resumable.to_json() 
1097          del d["http"] 
1098          del d["postproc"] 
1099          del d["_sleep"] 
1100          del d["_rand"] 
1101   
1102          return json.dumps(d) 
 1103   
1104      @staticmethod 
1106          """Returns an HttpRequest populated with info from a JSON object.""" 
1107          d = json.loads(s) 
1108          if d["resumable"] is not None: 
1109              d["resumable"] = MediaUpload.new_from_json(d["resumable"]) 
1110          return HttpRequest( 
1111              http, 
1112              postproc, 
1113              uri=d["uri"], 
1114              method=d["method"], 
1115              body=d["body"], 
1116              headers=d["headers"], 
1117              methodId=d["methodId"], 
1118              resumable=d["resumable"], 
1119          ) 
 1120   
1121      @staticmethod 
1122 -    def null_postproc(resp, contents): 
 1123          return resp, contents 
  1124   
1127      """Batches multiple HttpRequest objects into a single HTTP request. 
1128   
1129    Example: 
1130      from googleapiclient.http import BatchHttpRequest 
1131   
1132      def list_animals(request_id, response, exception): 
1133        \"\"\"Do something with the animals list response.\"\"\" 
1134        if exception is not None: 
1135          # Do something with the exception. 
1136          pass 
1137        else: 
1138          # Do something with the response. 
1139          pass 
1140   
1141      def list_farmers(request_id, response, exception): 
1142        \"\"\"Do something with the farmers list response.\"\"\" 
1143        if exception is not None: 
1144          # Do something with the exception. 
1145          pass 
1146        else: 
1147          # Do something with the response. 
1148          pass 
1149   
1150      service = build('farm', 'v2') 
1151   
1152      batch = BatchHttpRequest() 
1153   
1154      batch.add(service.animals().list(), list_animals) 
1155      batch.add(service.farmers().list(), list_farmers) 
1156      batch.execute(http=http) 
1157    """ 
1158   
1159      @util.positional(1) 
1160 -    def __init__(self, callback=None, batch_uri=None): 
 1161          """Constructor for a BatchHttpRequest. 
1162   
1163      Args: 
1164        callback: callable, A callback to be called for each response, of the 
1165          form callback(id, response, exception). The first parameter is the 
1166          request id, and the second is the deserialized response object. The 
1167          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1168          occurred while processing the request, or None if no error occurred. 
1169        batch_uri: string, URI to send batch requests to. 
1170      """ 
1171          if batch_uri is None: 
1172              batch_uri = _LEGACY_BATCH_URI 
1173   
1174          if batch_uri == _LEGACY_BATCH_URI: 
1175              LOGGER.warning( 
1176                  "You have constructed a BatchHttpRequest using the legacy batch " 
1177                  "endpoint %s. This endpoint will be turned down on August 12, 2020. " 
1178                  "Please provide the API-specific endpoint or use " 
1179                  "service.new_batch_http_request(). For more details see " 
1180                  "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" 
1181                  "and https://developers.google.com/api-client-library/python/guide/batch.", 
1182                  _LEGACY_BATCH_URI, 
1183              ) 
1184          self._batch_uri = batch_uri 
1185   
1186           
1187          self._callback = callback 
1188   
1189           
1190          self._requests = {} 
1191   
1192           
1193          self._callbacks = {} 
1194   
1195           
1196          self._order = [] 
1197   
1198           
1199          self._last_auto_id = 0 
1200   
1201           
1202          self._base_id = None 
1203   
1204           
1205          self._responses = {} 
1206   
1207           
1208          self._refreshed_credentials = {} 
 1209   
1211          """Refresh the credentials and apply to the request. 
1212   
1213      Args: 
1214        request: HttpRequest, the request. 
1215        http: httplib2.Http, the global http object for the batch. 
1216      """ 
1217           
1218           
1219           
1220          creds = None 
1221          request_credentials = False 
1222   
1223          if request.http is not None: 
1224              creds = _auth.get_credentials_from_http(request.http) 
1225              request_credentials = True 
1226   
1227          if creds is None and http is not None: 
1228              creds = _auth.get_credentials_from_http(http) 
1229   
1230          if creds is not None: 
1231              if id(creds) not in self._refreshed_credentials: 
1232                  _auth.refresh_credentials(creds) 
1233                  self._refreshed_credentials[id(creds)] = 1 
1234   
1235           
1236           
1237          if request.http is None or not request_credentials: 
1238              _auth.apply_credentials(creds, request.headers) 
 1239   
1241          """Convert an id to a Content-ID header value. 
1242   
1243      Args: 
1244        id_: string, identifier of individual request. 
1245   
1246      Returns: 
1247        A Content-ID header with the id_ encoded into it. A UUID is prepended to 
1248        the value because Content-ID headers are supposed to be universally 
1249        unique. 
1250      """ 
1251          if self._base_id is None: 
1252              self._base_id = uuid.uuid4() 
1253   
1254           
1255           
1256           
1257          return "<%s + %s>" % (self._base_id, quote(id_)) 
 1258   
1260          """Convert a Content-ID header value to an id. 
1261   
1262      Presumes the Content-ID header conforms to the format that _id_to_header() 
1263      returns. 
1264   
1265      Args: 
1266        header: string, Content-ID header value. 
1267   
1268      Returns: 
1269        The extracted id value. 
1270   
1271      Raises: 
1272        BatchError if the header is not in the expected format. 
1273      """ 
1274          if header[0] != "<" or header[-1] != ">": 
1275              raise BatchError("Invalid value for Content-ID: %s" % header) 
1276          if "+" not in header: 
1277              raise BatchError("Invalid value for Content-ID: %s" % header) 
1278          base, id_ = header[1:-1].split(" + ", 1) 
1279   
1280          return unquote(id_) 
 1281   
1283          """Convert an HttpRequest object into a string. 
1284   
1285      Args: 
1286        request: HttpRequest, the request to serialize. 
1287   
1288      Returns: 
1289        The request as a string in application/http format. 
1290      """ 
1291           
1292          parsed = urlparse(request.uri) 
1293          request_line = urlunparse( 
1294              ("", "", parsed.path, parsed.params, parsed.query, "") 
1295          ) 
1296          status_line = request.method + " " + request_line + " HTTP/1.1\n" 
1297          major, minor = request.headers.get("content-type", "application/json").split( 
1298              "/" 
1299          ) 
1300          msg = MIMENonMultipart(major, minor) 
1301          headers = request.headers.copy() 
1302   
1303          if request.http is not None: 
1304              credentials = _auth.get_credentials_from_http(request.http) 
1305              if credentials is not None: 
1306                  _auth.apply_credentials(credentials, headers) 
1307   
1308           
1309          if "content-type" in headers: 
1310              del headers["content-type"] 
1311   
1312          for key, value in six.iteritems(headers): 
1313              msg[key] = value 
1314          msg["Host"] = parsed.netloc 
1315          msg.set_unixfrom(None) 
1316   
1317          if request.body is not None: 
1318              msg.set_payload(request.body) 
1319              msg["content-length"] = str(len(request.body)) 
1320   
1321           
1322          fp = StringIO() 
1323           
1324          g = Generator(fp, maxheaderlen=0) 
1325          g.flatten(msg, unixfrom=False) 
1326          body = fp.getvalue() 
1327   
1328          return status_line + body 
 1329   
1331          """Convert string into httplib2 response and content. 
1332   
1333      Args: 
1334        payload: string, headers and body as a string. 
1335   
1336      Returns: 
1337        A pair (resp, content), such as would be returned from httplib2.request. 
1338      """ 
1339           
1340          status_line, payload = payload.split("\n", 1) 
1341          protocol, status, reason = status_line.split(" ", 2) 
1342   
1343           
1344          parser = FeedParser() 
1345          parser.feed(payload) 
1346          msg = parser.close() 
1347          msg["status"] = status 
1348   
1349           
1350          resp = httplib2.Response(msg) 
1351          resp.reason = reason 
1352          resp.version = int(protocol.split("/", 1)[1].replace(".", "")) 
1353   
1354          content = payload.split("\r\n\r\n", 1)[1] 
1355   
1356          return resp, content 
 1357   
1359          """Create a new id. 
1360   
1361      Auto incrementing number that avoids conflicts with ids already used. 
1362   
1363      Returns: 
1364         string, a new unique id. 
1365      """ 
1366          self._last_auto_id += 1 
1367          while str(self._last_auto_id) in self._requests: 
1368              self._last_auto_id += 1 
1369          return str(self._last_auto_id) 
 1370   
1371      @util.positional(2) 
1372 -    def add(self, request, callback=None, request_id=None): 
 1373          """Add a new request. 
1374   
1375      Every callback added will be paired with a unique id, the request_id. That 
1376      unique id will be passed back to the callback when the response comes back 
1377      from the server. The default behavior is to have the library generate it's 
1378      own unique id. If the caller passes in a request_id then they must ensure 
1379      uniqueness for each request_id, and if they are not an exception is 
1380      raised. Callers should either supply all request_ids or never supply a 
1381      request id, to avoid such an error. 
1382   
1383      Args: 
1384        request: HttpRequest, Request to add to the batch. 
1385        callback: callable, A callback to be called for this response, of the 
1386          form callback(id, response, exception). The first parameter is the 
1387          request id, and the second is the deserialized response object. The 
1388          third is an googleapiclient.errors.HttpError exception object if an HTTP error 
1389          occurred while processing the request, or None if no errors occurred. 
1390        request_id: string, A unique id for the request. The id will be passed 
1391          to the callback with the response. 
1392   
1393      Returns: 
1394        None 
1395   
1396      Raises: 
1397        BatchError if a media request is added to a batch. 
1398        KeyError is the request_id is not unique. 
1399      """ 
1400   
1401          if len(self._order) >= MAX_BATCH_LIMIT: 
1402              raise BatchError( 
1403                  "Exceeded the maximum calls(%d) in a single batch request." 
1404                  % MAX_BATCH_LIMIT 
1405              ) 
1406          if request_id is None: 
1407              request_id = self._new_id() 
1408          if request.resumable is not None: 
1409              raise BatchError("Media requests cannot be used in a batch request.") 
1410          if request_id in self._requests: 
1411              raise KeyError("A request with this ID already exists: %s" % request_id) 
1412          self._requests[request_id] = request 
1413          self._callbacks[request_id] = callback 
1414          self._order.append(request_id) 
 1415   
1416 -    def _execute(self, http, order, requests): 
 1417          """Serialize batch request, send to server, process response. 
1418   
1419      Args: 
1420        http: httplib2.Http, an http object to be used to make the request with. 
1421        order: list, list of request ids in the order they were added to the 
1422          batch. 
1423        requests: list, list of request objects to send. 
1424   
1425      Raises: 
1426        httplib2.HttpLib2Error if a transport error has occurred. 
1427        googleapiclient.errors.BatchError if the response is the wrong format. 
1428      """ 
1429          message = MIMEMultipart("mixed") 
1430           
1431          setattr(message, "_write_headers", lambda self: None) 
1432   
1433           
1434          for request_id in order: 
1435              request = requests[request_id] 
1436   
1437              msg = MIMENonMultipart("application", "http") 
1438              msg["Content-Transfer-Encoding"] = "binary" 
1439              msg["Content-ID"] = self._id_to_header(request_id) 
1440   
1441              body = self._serialize_request(request) 
1442              msg.set_payload(body) 
1443              message.attach(msg) 
1444   
1445           
1446           
1447          fp = StringIO() 
1448          g = Generator(fp, mangle_from_=False) 
1449          g.flatten(message, unixfrom=False) 
1450          body = fp.getvalue() 
1451   
1452          headers = {} 
1453          headers["content-type"] = ( 
1454              "multipart/mixed; " 'boundary="%s"' 
1455          ) % message.get_boundary() 
1456   
1457          resp, content = http.request( 
1458              self._batch_uri, method="POST", body=body, headers=headers 
1459          ) 
1460   
1461          if resp.status >= 300: 
1462              raise HttpError(resp, content, uri=self._batch_uri) 
1463   
1464           
1465          header = "content-type: %s\r\n\r\n" % resp["content-type"] 
1466           
1467           
1468          if six.PY3: 
1469              content = content.decode("utf-8") 
1470          for_parser = header + content 
1471   
1472          parser = FeedParser() 
1473          parser.feed(for_parser) 
1474          mime_response = parser.close() 
1475   
1476          if not mime_response.is_multipart(): 
1477              raise BatchError( 
1478                  "Response not in multipart/mixed format.", resp=resp, content=content 
1479              ) 
1480   
1481          for part in mime_response.get_payload(): 
1482              request_id = self._header_to_id(part["Content-ID"]) 
1483              response, content = self._deserialize_response(part.get_payload()) 
1484               
1485              if isinstance(content, six.text_type): 
1486                  content = content.encode("utf-8") 
1487              self._responses[request_id] = (response, content) 
 1488   
1489      @util.positional(1) 
1491          """Execute all the requests as a single batched HTTP request. 
1492   
1493      Args: 
1494        http: httplib2.Http, an http object to be used in place of the one the 
1495          HttpRequest request object was constructed with. If one isn't supplied 
1496          then use a http object from the requests in this batch. 
1497   
1498      Returns: 
1499        None 
1500   
1501      Raises: 
1502        httplib2.HttpLib2Error if a transport error has occurred. 
1503        googleapiclient.errors.BatchError if the response is the wrong format. 
1504      """ 
1505           
1506          if len(self._order) == 0: 
1507              return None 
1508   
1509           
1510          if http is None: 
1511              for request_id in self._order: 
1512                  request = self._requests[request_id] 
1513                  if request is not None: 
1514                      http = request.http 
1515                      break 
1516   
1517          if http is None: 
1518              raise ValueError("Missing a valid http object.") 
1519   
1520           
1521           
1522          creds = _auth.get_credentials_from_http(http) 
1523          if creds is not None: 
1524              if not _auth.is_valid(creds): 
1525                  LOGGER.info("Attempting refresh to obtain initial access_token") 
1526                  _auth.refresh_credentials(creds) 
1527   
1528          self._execute(http, self._order, self._requests) 
1529   
1530           
1531           
1532          redo_requests = {} 
1533          redo_order = [] 
1534   
1535          for request_id in self._order: 
1536              resp, content = self._responses[request_id] 
1537              if resp["status"] == "401": 
1538                  redo_order.append(request_id) 
1539                  request = self._requests[request_id] 
1540                  self._refresh_and_apply_credentials(request, http) 
1541                  redo_requests[request_id] = request 
1542   
1543          if redo_requests: 
1544              self._execute(http, redo_order, redo_requests) 
1545   
1546           
1547           
1548           
1549   
1550          for request_id in self._order: 
1551              resp, content = self._responses[request_id] 
1552   
1553              request = self._requests[request_id] 
1554              callback = self._callbacks[request_id] 
1555   
1556              response = None 
1557              exception = None 
1558              try: 
1559                  if resp.status >= 300: 
1560                      raise HttpError(resp, content, uri=request.uri) 
1561                  response = request.postproc(resp, content) 
1562              except HttpError as e: 
1563                  exception = e 
1564   
1565              if callback is not None: 
1566                  callback(request_id, response, exception) 
1567              if self._callback is not None: 
1568                  self._callback(request_id, response, exception) 
  1569   
1572      """Mock of HttpRequest. 
1573   
1574    Do not construct directly, instead use RequestMockBuilder. 
1575    """ 
1576   
1577 -    def __init__(self, resp, content, postproc): 
 1578          """Constructor for HttpRequestMock 
1579   
1580      Args: 
1581        resp: httplib2.Response, the response to emulate coming from the request 
1582        content: string, the response body 
1583        postproc: callable, the post processing function usually supplied by 
1584                  the model class. See model.JsonModel.response() as an example. 
1585      """ 
1586          self.resp = resp 
1587          self.content = content 
1588          self.postproc = postproc 
1589          if resp is None: 
1590              self.resp = httplib2.Response({"status": 200, "reason": "OK"}) 
1591          if "reason" in self.resp: 
1592              self.resp.reason = self.resp["reason"] 
 1593   
1595          """Execute the request. 
1596   
1597      Same behavior as HttpRequest.execute(), but the response is 
1598      mocked and not really from an HTTP request/response. 
1599      """ 
1600          return self.postproc(self.resp, self.content) 
  1601   
1604      """A simple mock of HttpRequest 
1605   
1606      Pass in a dictionary to the constructor that maps request methodIds to 
1607      tuples of (httplib2.Response, content, opt_expected_body) that should be 
1608      returned when that method is called. None may also be passed in for the 
1609      httplib2.Response, in which case a 200 OK response will be generated. 
1610      If an opt_expected_body (str or dict) is provided, it will be compared to 
1611      the body and UnexpectedBodyError will be raised on inequality. 
1612   
1613      Example: 
1614        response = '{"data": {"id": "tag:google.c...' 
1615        requestBuilder = RequestMockBuilder( 
1616          { 
1617            'plus.activities.get': (None, response), 
1618          } 
1619        ) 
1620        googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 
1621   
1622      Methods that you do not supply a response for will return a 
1623      200 OK with an empty string as the response content or raise an excpetion 
1624      if check_unexpected is set to True. The methodId is taken from the rpcName 
1625      in the discovery document. 
1626   
1627      For more details see the project wiki. 
1628    """ 
1629   
1630 -    def __init__(self, responses, check_unexpected=False): 
 1631          """Constructor for RequestMockBuilder 
1632   
1633      The constructed object should be a callable object 
1634      that can replace the class HttpResponse. 
1635   
1636      responses - A dictionary that maps methodIds into tuples 
1637                  of (httplib2.Response, content). The methodId 
1638                  comes from the 'rpcName' field in the discovery 
1639                  document. 
1640      check_unexpected - A boolean setting whether or not UnexpectedMethodError 
1641                         should be raised on unsupplied method. 
1642      """ 
1643          self.responses = responses 
1644          self.check_unexpected = check_unexpected 
 1645   
1646 -    def __call__( 
1647          self, 
1648          http, 
1649          postproc, 
1650          uri, 
1651          method="GET", 
1652          body=None, 
1653          headers=None, 
1654          methodId=None, 
1655          resumable=None, 
1656      ): 
 1657          """Implements the callable interface that discovery.build() expects 
1658      of requestBuilder, which is to build an object compatible with 
1659      HttpRequest.execute(). See that method for the description of the 
1660      parameters and the expected response. 
1661      """ 
1662          if methodId in self.responses: 
1663              response = self.responses[methodId] 
1664              resp, content = response[:2] 
1665              if len(response) > 2: 
1666                   
1667                  expected_body = response[2] 
1668                  if bool(expected_body) != bool(body): 
1669                       
1670                       
1671                      raise UnexpectedBodyError(expected_body, body) 
1672                  if isinstance(expected_body, str): 
1673                      expected_body = json.loads(expected_body) 
1674                  body = json.loads(body) 
1675                  if body != expected_body: 
1676                      raise UnexpectedBodyError(expected_body, body) 
1677              return HttpRequestMock(resp, content, postproc) 
1678          elif self.check_unexpected: 
1679              raise UnexpectedMethodError(methodId=methodId) 
1680          else: 
1681              model = JsonModel(False) 
1682              return HttpRequestMock(None, "{}", model.response) 
  1683   
1686      """Mock of httplib2.Http""" 
1687   
1688 -    def __init__(self, filename=None, headers=None): 
 1689          """ 
1690      Args: 
1691        filename: string, absolute filename to read response from 
1692        headers: dict, header to return with response 
1693      """ 
1694          if headers is None: 
1695              headers = {"status": "200"} 
1696          if filename: 
1697              with open(filename, "rb") as f: 
1698                  self.data = f.read() 
1699          else: 
1700              self.data = None 
1701          self.response_headers = headers 
1702          self.headers = None 
1703          self.uri = None 
1704          self.method = None 
1705          self.body = None 
1706          self.headers = None 
 1707   
1708 -    def request( 
1709          self, 
1710          uri, 
1711          method="GET", 
1712          body=None, 
1713          headers=None, 
1714          redirections=1, 
1715          connection_type=None, 
1716      ): 
 1717          self.uri = uri 
1718          self.method = method 
1719          self.body = body 
1720          self.headers = headers 
1721          return httplib2.Response(self.response_headers), self.data 
 1722   
 1725   
1727      """Mock of httplib2.Http 
1728   
1729    Mocks a sequence of calls to request returning different responses for each 
1730    call. Create an instance initialized with the desired response headers 
1731    and content and then use as if an httplib2.Http instance. 
1732   
1733      http = HttpMockSequence([ 
1734        ({'status': '401'}, ''), 
1735        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 
1736        ({'status': '200'}, 'echo_request_headers'), 
1737        ]) 
1738      resp, content = http.request("http://examples.com") 
1739   
1740    There are special values you can pass in for content to trigger 
1741    behavours that are helpful in testing. 
1742   
1743    'echo_request_headers' means return the request headers in the response body 
1744    'echo_request_headers_as_json' means return the request headers in 
1745       the response body 
1746    'echo_request_body' means return the request body in the response body 
1747    'echo_request_uri' means return the request uri in the response body 
1748    """ 
1749   
1751          """ 
1752      Args: 
1753        iterable: iterable, a sequence of pairs of (headers, body) 
1754      """ 
1755          self._iterable = iterable 
1756          self.follow_redirects = True 
1757          self.request_sequence = list() 
 1758   
1759 -    def request( 
1760          self, 
1761          uri, 
1762          method="GET", 
1763          body=None, 
1764          headers=None, 
1765          redirections=1, 
1766          connection_type=None, 
1767      ): 
 1768           
1769          self.request_sequence.append((uri, method, body, headers)) 
1770          resp, content = self._iterable.pop(0) 
1771          content = six.ensure_binary(content) 
1772   
1773          if content == b"echo_request_headers": 
1774              content = headers 
1775          elif content == b"echo_request_headers_as_json": 
1776              content = json.dumps(headers) 
1777          elif content == b"echo_request_body": 
1778              if hasattr(body, "read"): 
1779                  content = body.read() 
1780              else: 
1781                  content = body 
1782          elif content == b"echo_request_uri": 
1783              content = uri 
1784          if isinstance(content, six.text_type): 
1785              content = content.encode("utf-8") 
1786          return httplib2.Response(resp), content 
  1787   
1790      """Set the user-agent on every request. 
1791   
1792    Args: 
1793       http - An instance of httplib2.Http 
1794           or something that acts like it. 
1795       user_agent: string, the value for the user-agent header. 
1796   
1797    Returns: 
1798       A modified instance of http that was passed in. 
1799   
1800    Example: 
1801   
1802      h = httplib2.Http() 
1803      h = set_user_agent(h, "my-app-name/6.0") 
1804   
1805    Most of the time the user-agent will be set doing auth, this is for the rare 
1806    cases where you are accessing an unauthenticated endpoint. 
1807    """ 
1808      request_orig = http.request 
1809   
1810       
1811      def new_request( 
1812          uri, 
1813          method="GET", 
1814          body=None, 
1815          headers=None, 
1816          redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1817          connection_type=None, 
1818      ): 
1819          """Modify the request headers to add the user-agent.""" 
1820          if headers is None: 
1821              headers = {} 
1822          if "user-agent" in headers: 
1823              headers["user-agent"] = user_agent + " " + headers["user-agent"] 
1824          else: 
1825              headers["user-agent"] = user_agent 
1826          resp, content = request_orig( 
1827              uri, 
1828              method=method, 
1829              body=body, 
1830              headers=headers, 
1831              redirections=redirections, 
1832              connection_type=connection_type, 
1833          ) 
1834          return resp, content 
 1835   
1836      http.request = new_request 
1837      return http 
1838   
1841      """Tunnel PATCH requests over POST. 
1842    Args: 
1843       http - An instance of httplib2.Http 
1844           or something that acts like it. 
1845   
1846    Returns: 
1847       A modified instance of http that was passed in. 
1848   
1849    Example: 
1850   
1851      h = httplib2.Http() 
1852      h = tunnel_patch(h, "my-app-name/6.0") 
1853   
1854    Useful if you are running on a platform that doesn't support PATCH. 
1855    Apply this last if you are using OAuth 1.0, as changing the method 
1856    will result in a different signature. 
1857    """ 
1858      request_orig = http.request 
1859   
1860       
1861      def new_request( 
1862          uri, 
1863          method="GET", 
1864          body=None, 
1865          headers=None, 
1866          redirections=httplib2.DEFAULT_MAX_REDIRECTS, 
1867          connection_type=None, 
1868      ): 
1869          """Modify the request headers to add the user-agent.""" 
1870          if headers is None: 
1871              headers = {} 
1872          if method == "PATCH": 
1873              if "oauth_token" in headers.get("authorization", ""): 
1874                  LOGGER.warning( 
1875                      "OAuth 1.0 request made with Credentials after tunnel_patch." 
1876                  ) 
1877              headers["x-http-method-override"] = "PATCH" 
1878              method = "POST" 
1879          resp, content = request_orig( 
1880              uri, 
1881              method=method, 
1882              body=body, 
1883              headers=headers, 
1884              redirections=redirections, 
1885              connection_type=connection_type, 
1886          ) 
1887          return resp, content 
 1888   
1889      http.request = new_request 
1890      return http 
1891   
1894      """Builds httplib2.Http object 
1895   
1896    Returns: 
1897    A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 
1898    To override default timeout call 
1899   
1900      socket.setdefaulttimeout(timeout_in_sec) 
1901   
1902    before interacting with this method. 
1903    """ 
1904      if socket.getdefaulttimeout() is not None: 
1905          http_timeout = socket.getdefaulttimeout() 
1906      else: 
1907          http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 
1908      http = httplib2.Http(timeout=http_timeout) 
1909       
1910       
1911       
1912       
1913      try: 
1914        http.redirect_codes = http.redirect_codes - {308} 
1915      except AttributeError: 
1916         
1917         
1918         
1919         
1920        pass 
1921   
1922      return http 
 1923