Merge pull request #247 from dedsm/boto3_support

Adding support for boto3

.travis.yml

@@ -16,6 +16,7 @@ env:
     - TOX_SUFFIX="requests1"
     - TOX_SUFFIX="httplib2"
     - TOX_SUFFIX="boto"
+    - TOX_SUFFIX="boto3"
     - TOX_SUFFIX="urllib317"
     - TOX_SUFFIX="urllib319"
     - TOX_SUFFIX="urllib3110"
@@ -24,6 +25,7 @@ env:
 matrix:
   allow_failures:
     - env: TOX_SUFFIX="boto"
+    - env: TOX_SUFFIX="boto3"
   exclude:
     - env: TOX_SUFFIX="boto"
       python: 3.3

tests/integration/test_boto.py

@@ -5,8 +5,12 @@ import boto  # NOQA
 import boto.iam  # NOQA
 from boto.s3.connection import S3Connection  # NOQA
 from boto.s3.key import Key  # NOQA
-from ConfigParser import DuplicateSectionError  # NOQA
 import vcr  # NOQA
+try:  # NOQA
+    from ConfigParser import DuplicateSectionError  # NOQA
+except ImportError:  # NOQA
+    # python3
+    from configparser import DuplicateSectionError  # NOQA
 
 
 def test_boto_stubs(tmpdir):

tests/integration/test_boto3.py

@@ -0,0 +1,67 @@
+import pytest
+boto3 = pytest.importorskip("boto3")
+
+import boto3  # NOQA
+import vcr  # NOQA
+
+bucket = 'boto3-demo-1337'              # a bucket you can access
+key = 'test/my_test.txt'                # key with r+w access
+content = 'hello world i am a string'   # content to put in the test file
+
+
+def test_boto_stubs(tmpdir):
+    with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))):
+        # Perform the imports within the patched context so that
+        # HTTPConnection, VerifiedHTTPSConnection refers to the patched version.
+        from botocore.vendored.requests.packages.urllib3.connectionpool import \
+            HTTPConnection, VerifiedHTTPSConnection
+        from vcr.stubs.boto3_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection
+        # Prove that the class was patched by the stub and that we can instantiate it.
+        assert issubclass(HTTPConnection, VCRRequestsHTTPConnection)
+        assert issubclass(VerifiedHTTPSConnection, VCRRequestsHTTPSConnection)
+        HTTPConnection('hostname.does.not.matter')
+        VerifiedHTTPSConnection('hostname.does.not.matter')
+
+
+def test_boto3_without_vcr():
+    s3_resource = boto3.resource('s3')
+    b = s3_resource.Bucket(bucket)
+    b.put_object(Key=key, Body=content)
+
+    # retrieve content to check it
+    o = s3_resource.Object(bucket, key).get()
+
+    # decode for python3
+    assert content == o['Body'].read().decode('utf-8')
+
+
+def test_boto_medium_difficulty(tmpdir):
+    s3_resource = boto3.resource('s3')
+    b = s3_resource.Bucket(bucket)
+    with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))):
+        b.put_object(Key=key, Body=content)
+        o = s3_resource.Object(bucket, key).get()
+        assert content == o['Body'].read().decode('utf-8')
+
+    with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass:
+        b.put_object(Key=key, Body=content)
+        o = s3_resource.Object(bucket, key).get()
+        assert content == o['Body'].read().decode('utf-8')
+        assert cass.all_played
+
+
+def test_boto_hardcore_mode(tmpdir):
+    with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))):
+        s3_resource = boto3.resource('s3')
+        b = s3_resource.Bucket(bucket)
+        b.put_object(Key=key, Body=content)
+        o = s3_resource.Object(bucket, key).get()
+        assert content == o['Body'].read().decode('utf-8')
+
+    with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass:
+        s3_resource = boto3.resource('s3')
+        b = s3_resource.Bucket(bucket)
+        b.put_object(Key=key, Body=content)
+        o = s3_resource.Object(bucket, key).get()
+        assert content == o['Body'].read().decode('utf-8')
+        assert cass.all_played

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py26,py27,py33,py34,pypy,pypy3}-{flakes,requests27,requests26,requests25,requests24,requests23,requests22,requests1,httplib2,urllib317,urllib319,urllib3110,tornado3,tornado4,boto}
+envlist = {py26,py27,py33,py34,pypy,pypy3}-{flakes,requests27,requests26,requests25,requests24,requests23,requests22,requests1,httplib2,urllib317,urllib319,urllib3110,tornado3,tornado4,boto,boto3}
 
 [testenv:flakes]
 skipsdist = True
@@ -37,6 +37,7 @@ deps =
     {py26,py27,py33,py34}-tornado3: pycurl
     {py26,py27,py33,py34}-tornado4: pycurl
     boto: boto
+    boto3: boto3
 
 [flake8]
 max_line_length = 110

vcr/patch.py

@@ -22,6 +22,16 @@ else:
     _cpoolHTTPConnection = cpool.HTTPConnection
     _cpoolHTTPSConnection = cpool.HTTPSConnection
 
+# Try to save the original types for boto3
+try:
+    import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+except ImportError:  # pragma: no cover
+    pass
+else:
+    _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection
+    _cpoolBoto3HTTPConnection = cpool.HTTPConnection
+    _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection
+
 
 # Try to save the original types for urllib3
 try:
@@ -87,7 +97,7 @@ class CassettePatcherBuilder(object):
 
     def build(self):
         return itertools.chain(
-            self._httplib(), self._requests(), self._urllib3(),
+            self._httplib(), self._requests(), self._boto3(), self._urllib3(),
             self._httplib2(), self._boto(), self._tornado(),
             self._build_patchers_from_mock_triples(
                 self._cassette.custom_patches
@@ -165,6 +175,14 @@ class CassettePatcherBuilder(object):
         from .stubs import requests_stubs
         return self._urllib3_patchers(cpool, requests_stubs)
 
+    def _boto3(self):
+        try:
+            import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+        except ImportError:  # pragma: no cover
+            return ()
+        from .stubs import boto3_stubs
+        return self._urllib3_patchers(cpool, boto3_stubs)
+
     def _patched_get_conn(self, connection_pool_class, connection_class_getter):
         get_conn = connection_pool_class._get_conn
 
@@ -353,6 +371,24 @@ def reset_patchers():
             yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', _HTTPConnection)
             yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _HTTPSConnection)
 
+    try:
+        import botocore.vendored.requests.packages.urllib3.connectionpool as cpool
+    except ImportError:  # pragma: no cover
+        pass
+    else:
+        # unpatch requests v1.x
+        yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection)
+        yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection)
+        # unpatch requests v2.x
+        if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'):
+            yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls',
+                                    _cpoolBoto3HTTPConnection)
+            yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls',
+                                    _cpoolBoto3HTTPSConnection)
+
+        if hasattr(cpool, 'HTTPSConnection'):
+            yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection)
+
     try:
         import httplib2 as cpool
     except ImportError:  # pragma: no cover

vcr/stubs/boto3_stubs.py

@@ -0,0 +1,15 @@
+'''Stubs for boto3'''
+
+from botocore.vendored.requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
+from ..stubs import VCRHTTPConnection, VCRHTTPSConnection
+
+# urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes
+# you're using.  It includes some polyfills for newer features missing in older pythons.
+
+
+class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection):
+    _baseclass = HTTPConnection
+
+
+class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection):
+    _baseclass = VerifiedHTTPSConnection