1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Base Cloud API Client.
17
18BasicCloudApiCliend does basic setup for a cloud API.
19"""
20import logging
21import socket
22import ssl
23
24import six
25from six.moves import http_client
26
27# pylint: disable=import-error
28import httplib2
29from apiclient import errors as gerrors
30from apiclient.discovery import build
31from oauth2client import client
32
33from acloud import errors
34from acloud.internal.lib import utils
35
36
37logger = logging.getLogger(__name__)
38
39
40class BaseCloudApiClient(object):
41    """A class that does basic setup for a cloud API."""
42
43    # To be overriden by subclasses.
44    API_NAME = ""
45    API_VERSION = "v1"
46    SCOPE = ""
47
48    # Defaults for retry.
49    RETRY_COUNT = 5
50    RETRY_BACKOFF_FACTOR = 1.5
51    RETRY_SLEEP_MULTIPLIER = 2
52    RETRY_HTTP_CODES = [
53        # 403 is to retry the "Rate Limit Exceeded" error.
54        # We could retry on a finer-grained error message later if necessary.
55        403,
56        500,  # Internal Server Error
57        502,  # Bad Gateway
58        503,  # Service Unavailable
59    ]
60    RETRIABLE_ERRORS = (http_client.HTTPException, httplib2.HttpLib2Error,
61                        socket.error, ssl.SSLError)
62    RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
63
64    def __init__(self, oauth2_credentials):
65        """Initialize.
66
67        Args:
68            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
69        """
70        self._service = self.InitResourceHandle(oauth2_credentials)
71
72    @classmethod
73    def InitResourceHandle(cls, oauth2_credentials):
74        """Authenticate and initialize a Resource object.
75
76        Authenticate http and create a Resource object with methods
77        for interacting with the service.
78
79        Args:
80            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
81
82        Returns:
83            An apiclient.discovery.Resource object
84        """
85        http_auth = oauth2_credentials.authorize(httplib2.Http())
86        return utils.RetryExceptionType(
87            exception_types=cls.RETRIABLE_AUTH_ERRORS,
88            max_retries=cls.RETRY_COUNT,
89            functor=build,
90            sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
91            retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
92            serviceName=cls.API_NAME,
93            version=cls.API_VERSION,
94            # This is workaround for a known issue of some veriosn
95            # of api client.
96            # https://github.com/google/google-api-python-client/issues/435
97            cache_discovery=False,
98            http=http_auth)
99
100    @staticmethod
101    def _ShouldRetry(exception, retry_http_codes,
102                     other_retriable_errors):
103        """Check if exception is retriable.
104
105        Args:
106            exception: An instance of Exception.
107            retry_http_codes: a list of integers, retriable HTTP codes of
108                              HttpError
109            other_retriable_errors: a tuple of error types to retry other than
110                                    HttpError.
111
112        Returns:
113            Boolean, True if retriable, False otherwise.
114        """
115        if isinstance(exception, other_retriable_errors):
116            return True
117
118        if isinstance(exception, errors.HttpError):
119            if exception.code in retry_http_codes:
120                return True
121            logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
122                         exception.code, retry_http_codes, str(exception))
123
124        logger.debug("_ShouldRetry: Exception %s is not one of %s: %s",
125                     type(exception),
126                     list(other_retriable_errors) + [errors.HttpError],
127                     str(exception))
128        return False
129
130    @staticmethod
131    def _TranslateError(exception):
132        """Translate the exception to a desired type.
133
134        Args:
135            exception: An instance of Exception.
136
137        Returns:
138            gerrors.HttpError will be translated to errors.HttpError.
139            If the error code is errors.HTTP_NOT_FOUND_CODE, it will
140            be translated to errors.ResourceNotFoundError.
141            Unrecognized error type will not be translated and will
142            be returned as is.
143        """
144        if isinstance(exception, gerrors.HttpError):
145            exception = errors.HttpError.CreateFromHttpError(exception)
146            if exception.code == errors.HTTP_NOT_FOUND_CODE:
147                exception = errors.ResourceNotFoundError(
148                    exception.code, str(exception))
149        return exception
150
151    def ExecuteOnce(self, api):
152        """Execute an api and parse the errors.
153
154        Args:
155            api: An apiclient.http.HttpRequest, representing the api to execute.
156
157        Returns:
158            Execution result of the api.
159
160        Raises:
161            errors.ResourceNotFoundError: For 404 error.
162            errors.HttpError: For other types of http error.
163        """
164        try:
165            return api.execute()
166        except gerrors.HttpError as e:
167            raise self._TranslateError(e)
168
169    def Execute(self,
170                api,
171                retry_http_codes=None,
172                max_retry=None,
173                sleep=None,
174                backoff_factor=None,
175                other_retriable_errors=None):
176        """Execute an api with retry.
177
178        Call ExecuteOnce and retry on http error with given codes.
179
180        Args:
181            api: An apiclient.http.HttpRequest, representing the api to execute:
182            retry_http_codes: A list of http codes to retry.
183            max_retry: See utils.Retry.
184            sleep: See utils.Retry.
185            backoff_factor: See utils.Retry.
186            other_retriable_errors: A tuple of error types that should be retried
187                                    other than errors.HttpError.
188
189        Returns:
190          Execution result of the api.
191
192        Raises:
193          See ExecuteOnce.
194        """
195        retry_http_codes = (self.RETRY_HTTP_CODES
196                            if retry_http_codes is None else retry_http_codes)
197        max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
198        sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
199        backoff_factor = (self.RETRY_BACKOFF_FACTOR
200                          if backoff_factor is None else backoff_factor)
201        other_retriable_errors = (self.RETRIABLE_ERRORS
202                                  if other_retriable_errors is None else
203                                  other_retriable_errors)
204
205        def _Handler(exc):
206            """Check if |exc| is a retriable exception.
207
208            Args:
209                exc: An exception.
210
211            Returns:
212                True if exc is an errors.HttpError and code exists in |retry_http_codes|
213                False otherwise.
214            """
215            if self._ShouldRetry(exc, retry_http_codes,
216                                 other_retriable_errors):
217                logger.debug("Will retry error: %s", str(exc))
218                return True
219            return False
220
221        return utils.Retry(
222            _Handler,
223            max_retries=max_retry,
224            functor=self.ExecuteOnce,
225            sleep_multiplier=sleep,
226            retry_backoff_factor=backoff_factor,
227            api=api)
228
229    def BatchExecuteOnce(self, requests):
230        """Execute requests in a batch.
231
232        Args:
233            requests: A dictionary where key is request id and value
234                      is an http request.
235
236        Returns:
237            results, a dictionary in the following format
238            {request_id: (response, exception)}
239            request_ids are those from requests; response
240            is the http response for the request or None on error;
241            exception is an instance of DriverError or None if no error.
242        """
243        results = {}
244
245        def _CallBack(request_id, response, exception):
246            results[request_id] = (response, self._TranslateError(exception))
247
248        batch = self._service.new_batch_http_request()
249        for request_id, request in six.iteritems(requests):
250            batch.add(
251                request=request, callback=_CallBack, request_id=request_id)
252        batch.execute()
253        return results
254
255    def BatchExecute(self,
256                     requests,
257                     retry_http_codes=None,
258                     max_retry=None,
259                     sleep=None,
260                     backoff_factor=None,
261                     other_retriable_errors=None):
262        """Batch execute multiple requests with retry.
263
264        Call BatchExecuteOnce and retry on http error with given codes.
265
266        Args:
267            requests: A dictionary where key is request id picked by caller,
268                      and value is a apiclient.http.HttpRequest.
269            retry_http_codes: A list of http codes to retry.
270            max_retry: See utils.Retry.
271            sleep: See utils.Retry.
272            backoff_factor: See utils.Retry.
273            other_retriable_errors: A tuple of error types that should be retried
274                                    other than errors.HttpError.
275
276        Returns:
277            results, a dictionary in the following format
278            {request_id: (response, exception)}
279            request_ids are those from requests; response
280            is the http response for the request or None on error;
281            exception is an instance of DriverError or None if no error.
282        """
283        executor = utils.BatchHttpRequestExecutor(
284            self.BatchExecuteOnce,
285            requests=requests,
286            retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
287            max_retry=max_retry or self.RETRY_COUNT,
288            sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
289            backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
290            other_retriable_errors=other_retriable_errors
291            or self.RETRIABLE_ERRORS)
292        executor.Execute()
293        return executor.GetResults()
294
295    def ListWithMultiPages(self, api_resource, *args, **kwargs):
296        """Call an api that list a type of resource.
297
298        Multiple google services support listing a type of
299        resource (e.g list gce instances, list storage objects).
300        The querying pattern is similar --
301        Step 1: execute the api and get a response object like,
302        {
303            "items": [..list of resource..],
304            # The continuation token that can be used
305            # to get the next page.
306            "nextPageToken": "A String",
307        }
308        Step 2: execute the api again with the nextPageToken to
309        retrieve more pages and get a response object.
310
311        Step 3: Repeat Step 2 until no more page.
312
313        This method encapsulates the generic logic of
314        calling such listing api.
315
316        Args:
317            api_resource: An apiclient.discovery.Resource object
318                used to create an http request for the listing api.
319            *args: Arguments used to create the http request.
320            **kwargs: Keyword based arguments to create the http
321                      request.
322
323        Returns:
324            A list of items.
325        """
326        items = []
327        next_page_token = None
328        while True:
329            api = api_resource(pageToken=next_page_token, *args, **kwargs)
330            response = self.Execute(api)
331            items.extend(response.get("items", []))
332            next_page_token = response.get("nextPageToken")
333            if not next_page_token:
334                break
335        return items
336
337    @property
338    def service(self):
339        """Return self._service as a property."""
340        return self._service
341