Merge pull request #411 from jans-forks/fix/382-boto3

Fixes #382 boto3 compatibility.

.travis.yml

@@ -9,7 +9,7 @@ env:
     - TOX_SUFFIX="requests"
     - TOX_SUFFIX="httplib2"
     - TOX_SUFFIX="boto3"
-    - TOX_SUFFIX="urllib3121"
+    - TOX_SUFFIX="urllib3"
     - TOX_SUFFIX="tornado4"
     - TOX_SUFFIX="aiohttp"
 matrix:
@@ -23,7 +23,7 @@ matrix:
     - env: TOX_SUFFIX="httplib2"
       python: 3.7
       dist: xenial
-    - env: TOX_SUFFIX="urllib3121"
+    - env: TOX_SUFFIX="urllib3"
       python: 3.7
       dist: xenial
     - env: TOX_SUFFIX="tornado4"

tests/fixtures/migration/new_cassette.yaml

@@ -9,7 +9,7 @@ interactions:
     method: GET
     uri: http://httpbin.org/ip
   response:
-    body: {string: !!python/unicode "{\n  \"origin\": \"217.122.164.194\"\n}"}
+    body: {string: "{\n  \"origin\": \"217.122.164.194\"\n}"}
     headers:
       access-control-allow-origin: ['*']
       content-type: [application/json]

tests/integration/test_boto3.py

@@ -1,15 +1,61 @@
 import pytest
+import os
+
 boto3 = pytest.importorskip("boto3")
 
 import boto3  # NOQA
+import botocore  # NOQA
 import vcr  # NOQA
 
-bucket = 'boto3-demo-1337'              # a bucket you can access
+try:
-key = 'test/my_test.txt'                # key with r+w access
+    from botocore import awsrequest  # NOQA
-content = 'hello world i am a string'   # content to put in the test file
+
+    botocore_awsrequest = True
+except ImportError:
+    botocore_awsrequest = False
+
+# skip tests if boto does not use vendored requests anymore
+# https://github.com/boto/botocore/pull/1495
+boto3_skip_vendored_requests = pytest.mark.skipif(
+    botocore_awsrequest,
+    reason='botocore version {ver} does not use vendored requests anymore.'.format(
+        ver=botocore.__version__))
+
+boto3_skip_awsrequest = pytest.mark.skipif(
+    not botocore_awsrequest,
+    reason='botocore version {ver} still uses vendored requests.'.format(
+        ver=botocore.__version__))
 
 
-def test_boto_stubs(tmpdir):
+IAM_USER_NAME = "vcrpy"
+
+
+@pytest.fixture
+def iam_client():
+    def _iam_client(boto3_session=None):
+        if boto3_session is None:
+            boto3_session = boto3.Session(
+                aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', "default"),
+                aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY', "default"),
+                aws_session_token=None,
+                region_name=os.environ.get('AWS_DEFAULT_REGION', "default"),
+            )
+        return boto3_session.client('iam')
+    return _iam_client
+
+
+@pytest.fixture
+def get_user(iam_client):
+    def _get_user(client=None, user_name=IAM_USER_NAME):
+        if client is None:
+            # Default client set with fixture `iam_client`
+            client = iam_client()
+        return client.get_user(UserName=user_name)
+    return _get_user
+
+
+@boto3_skip_vendored_requests
+def test_boto_vendored_stubs(tmpdir):
     with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))):
         # Perform the imports within the patched context so that
         # HTTPConnection, VerifiedHTTPSConnection refers to the patched version.
@@ -23,45 +69,38 @@ def test_boto_stubs(tmpdir):
         VerifiedHTTPSConnection('hostname.does.not.matter')
 
 
-def test_boto3_without_vcr():
+def test_boto_medium_difficulty(tmpdir, get_user):
-    s3_resource = boto3.resource('s3')
-    b = s3_resource.Bucket(bucket)
-    b.put_object(Key=key, Body=content)
 
-    # retrieve content to check it
-    o = s3_resource.Object(bucket, key).get()
-
-    # decode for python3
-    assert content == o['Body'].read().decode('utf-8')
-
-
-def test_boto_medium_difficulty(tmpdir):
-    s3_resource = boto3.resource('s3')
-    b = s3_resource.Bucket(bucket)
     with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))):
-        b.put_object(Key=key, Body=content)
+        response = get_user()
-        o = s3_resource.Object(bucket, key).get()
+        assert response['User']['UserName'] == IAM_USER_NAME
-        assert content == o['Body'].read().decode('utf-8')
 
     with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass:
-        b.put_object(Key=key, Body=content)
+        response = get_user()
-        o = s3_resource.Object(bucket, key).get()
+        assert response['User']['UserName'] == IAM_USER_NAME
-        assert content == o['Body'].read().decode('utf-8')
         assert cass.all_played
 
 
-def test_boto_hardcore_mode(tmpdir):
+def test_boto_hardcore_mode(tmpdir, iam_client, get_user):
     with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))):
-        s3_resource = boto3.resource('s3')
+        ses = boto3.Session(
-        b = s3_resource.Bucket(bucket)
+            aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
-        b.put_object(Key=key, Body=content)
+            aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'),
-        o = s3_resource.Object(bucket, key).get()
+            region_name=os.environ.get('AWS_DEFAULT_REGION'),
-        assert content == o['Body'].read().decode('utf-8')
+        )
+        client = iam_client(ses)
+        response = get_user(client=client)
+        assert response['User']['UserName'] == IAM_USER_NAME
 
     with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass:
-        s3_resource = boto3.resource('s3')
+        ses = boto3.Session(
-        b = s3_resource.Bucket(bucket)
+            aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
-        b.put_object(Key=key, Body=content)
+            aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'),
-        o = s3_resource.Object(bucket, key).get()
+            aws_session_token=None,
-        assert content == o['Body'].read().decode('utf-8')
+            region_name=os.environ.get('AWS_DEFAULT_REGION'),
+        )
+
+        client = iam_client(ses)
+        response = get_user(client=client)
+        assert response['User']['UserName'] == IAM_USER_NAME
         assert cass.all_played

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3121,tornado4,boto3},{py35,py36,py37}-{aiohttp}
+envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp}
 
 [testenv:flakes]
 skipsdist = True
@@ -21,14 +21,18 @@ deps =
     ipaddress
     requests: requests>=2.22.0
     httplib2: httplib2
-    urllib3121: urllib3==1.21.1
+    urllib3: urllib3
     {py27,py35,py36,pypy}-tornado4: tornado>=4,<5
     {py27,py35,py36,pypy}-tornado4: pytest-tornado
     {py27,py35,py36}-tornado4: pycurl
     boto3: boto3
+    boto3: urllib3
     aiohttp: aiohttp
     aiohttp: pytest-asyncio
     aiohttp: pytest-aiohttp
-
+passenv =
+    AWS_ACCESS_KEY_ID
+    AWS_DEFAULT_REGION
+    AWS_SECRET_ACCESS_KEY
 [flake8]
 max_line_length = 110

vcr/cassette.py

@@ -190,6 +190,7 @@ class Cassette(object):
         self._serializer = serializer or yamlserializer
         self._match_on = match_on
         self._before_record_request = before_record_request or (lambda x: x)
+        log.info(self._before_record_request)
         self._before_record_response = before_record_response or (lambda x: x)
         self.inject = inject
         self.record_mode = record_mode
@@ -225,6 +226,7 @@ class Cassette(object):
 
     def append(self, request, response):
         """Add a request, response pair to this cassette"""
+        log.info("Appending request %s and response %s", request, response)
         request = self._before_record_request(request)
         if not request:
             return

vcr/patch.py

@@ -6,21 +6,29 @@ from .compat import contextlib, mock
 from .stubs import VCRHTTPConnection, VCRHTTPSConnection
 from six.moves import http_client as httplib
 
+import logging
 
+log = logging.getLogger(__name__)
 # Save some of the original types for the purposes of unpatching
 _HTTPConnection = httplib.HTTPConnection
 _HTTPSConnection = httplib.HTTPSConnection
 
-
 # Try to save the original types for boto3
 try:
-    import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+    from botocore.awsrequest import AWSHTTPSConnection, AWSHTTPConnection
-except ImportError:  # pragma: no cover
+except ImportError:
-    pass
+    try:
+        import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+    except ImportError:  # pragma: no cover
+        pass
+    else:
+        _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection
+        _cpoolBoto3HTTPConnection = cpool.HTTPConnection
+        _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection
 else:
-    _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection
+    _Boto3VerifiedHTTPSConnection = AWSHTTPSConnection
-    _cpoolBoto3HTTPConnection = cpool.HTTPConnection
+    _cpoolBoto3HTTPConnection = AWSHTTPConnection
-    _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection
+    _cpoolBoto3HTTPSConnection = AWSHTTPSConnection
 
 cpool = None
 # Try to save the original types for urllib3
@@ -44,7 +52,6 @@ else:
     _cpoolHTTPConnection = cpool.HTTPConnection
     _cpoolHTTPSConnection = cpool.HTTPSConnection
 
-
 # Try to save the original types for httplib2
 try:
     import httplib2
@@ -55,7 +62,6 @@ else:
     _HTTPSConnectionWithTimeout = httplib2.HTTPSConnectionWithTimeout
     _SCHEME_TO_CONNECTION = httplib2.SCHEME_TO_CONNECTION
 
-
 # Try to save the original types for boto
 try:
     import boto.https_connection
@@ -64,7 +70,6 @@ except ImportError:  # pragma: no cover
 else:
     _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection
 
-
 # Try to save the original types for Tornado
 try:
     import tornado.simple_httpclient
@@ -74,7 +79,6 @@ else:
     _SimpleAsyncHTTPClient_fetch_impl = \
         tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch_impl
 
-
 try:
     import tornado.curl_httpclient
 except ImportError:  # pragma: no cover
@@ -99,6 +103,7 @@ class CassettePatcherBuilder(object):
             return self._build_patchers_from_mock_triples(
                 function(self, *args, **kwargs)
             )
+
         return wrapped
 
     def __init__(self, cassette):
@@ -184,13 +189,26 @@ class CassettePatcherBuilder(object):
             return ()
         return self._urllib3_patchers(cpool, requests_stubs)
 
+    @_build_patchers_from_mock_triples_decorator
     def _boto3(self):
+
         try:
-            import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+            # botocore using awsrequest
+            import botocore.awsrequest as cpool
         except ImportError:  # pragma: no cover
-            return ()
+            try:
-        from .stubs import boto3_stubs
+                # botocore using vendored requests
-        return self._urllib3_patchers(cpool, boto3_stubs)
+                import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+            except ImportError:  # pragma: no cover
+                pass
+            else:
+                from .stubs import boto3_stubs
+                yield self._urllib3_patchers(cpool, boto3_stubs)
+        else:
+            from .stubs import boto3_stubs
+            log.debug("Patching boto3 cpool with %s", cpool)
+            yield cpool.AWSHTTPConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPConnection
+            yield cpool.AWSHTTPSConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPSConnection
 
     def _patched_get_conn(self, connection_pool_class, connection_class_getter):
         get_conn = connection_pool_class._get_conn
@@ -407,22 +425,36 @@ def reset_patchers():
             yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _cpoolHTTPSConnection)
 
     try:
-        import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+        # unpatch botocore with awsrequest
+        import botocore.awsrequest as cpool
     except ImportError:  # pragma: no cover
-        pass
+        try:
+            # unpatch botocore with vendored requests
+            import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+        except ImportError:  # pragma: no cover
+            pass
+        else:
+            # unpatch requests v1.x
+            yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection)
+            yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection)
+            # unpatch requests v2.x
+            if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'):
+                yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls',
+                                        _cpoolBoto3HTTPConnection)
+                yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls',
+                                        _cpoolBoto3HTTPSConnection)
+
+            if hasattr(cpool, 'HTTPSConnection'):
+                yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection)
     else:
-        # unpatch requests v1.x
+        if hasattr(cpool.AWSHTTPConnectionPool, 'ConnectionCls'):
-        yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection)
+            yield mock.patch.object(cpool.AWSHTTPConnectionPool, 'ConnectionCls',
-        yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection)
-        # unpatch requests v2.x
-        if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'):
-            yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls',
                                     _cpoolBoto3HTTPConnection)
-            yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls',
+            yield mock.patch.object(cpool.AWSHTTPSConnectionPool, 'ConnectionCls',
                                     _cpoolBoto3HTTPSConnection)
 
-        if hasattr(cpool, 'HTTPSConnection'):
+        if hasattr(cpool, 'AWSHTTPSConnection'):
-            yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection)
+            yield mock.patch.object(cpool, 'AWSHTTPSConnection', _cpoolBoto3HTTPSConnection)
 
     try:
         import httplib2 as cpool

vcr/request.py

@@ -2,6 +2,9 @@ import warnings
 from six import BytesIO, text_type
 from six.moves.urllib.parse import urlparse, parse_qsl
 from .util import CaseInsensitiveDict
+import logging
+
+log = logging.getLogger(__name__)
 
 
 class Request(object):
@@ -18,6 +21,7 @@ class Request(object):
         else:
             self.body = body
         self.headers = headers
+        log.debug("Invoking Request %s", self.uri)
 
     @property
     def headers(self):

vcr/stubs/__init__.py

@@ -170,6 +170,7 @@ class VCRConnection(object):
             self._port_postfix(),
             url,
         )
+        log.debug("Absolute URI: %s", uri)
         return uri
 
     def _url(self, uri):

vcr/stubs/boto3_stubs.py

@@ -1,11 +1,22 @@
-'''Stubs for boto3'''
+"""Stubs for boto3"""
+import six
+
+try:
+    # boto using awsrequest
+    from botocore.awsrequest import AWSHTTPConnection as HTTPConnection
+    from botocore.awsrequest import AWSHTTPSConnection as VerifiedHTTPSConnection
+
+except ImportError:  # pragma: nocover
+    # boto using vendored requests
+    # urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes
+    # you're using.  It includes some polyfills for newer features missing in older pythons.
+    try:
+        from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
+    except ImportError:  # pragma: nocover
+        from requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
 
-from botocore.vendored.requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
 from ..stubs import VCRHTTPConnection, VCRHTTPSConnection
 
-# urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes
-# you're using.  It includes some polyfills for newer features missing in older pythons.
-
 
 class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection):
     _baseclass = HTTPConnection
@@ -13,3 +24,20 @@ class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection):
 
 class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection):
     _baseclass = VerifiedHTTPSConnection
+
+    def __init__(self, *args, **kwargs):
+        if six.PY3:
+            kwargs.pop('strict', None)  # apparently this is gone in py3
+
+        # need to temporarily reset here because the real connection
+        # inherits from the thing that we are mocking out.  Take out
+        # the reset if you want to see what I mean :)
+        from vcr.patch import force_reset
+        with force_reset():
+            self.real_connection = self._baseclass(*args, **kwargs)
+            # Make sure to set those attributes as it seems `AWSHTTPConnection` does not
+            # set them, making the connection to fail !
+            self.real_connection.assert_hostname = kwargs.get("assert_hostname", False)
+            self.real_connection.cert_reqs = kwargs.get("cert_reqs", 'CERT_NONE')
+
+        self._sock = None