diff --git a/.travis.yml b/.travis.yml index 4988390..e26e4e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: - TOX_SUFFIX="httplib2" - TOX_SUFFIX="boto3" - TOX_SUFFIX="urllib3121" + - TOX_SUFFIX="urllib3" - TOX_SUFFIX="tornado4" - TOX_SUFFIX="aiohttp" matrix: @@ -45,9 +46,9 @@ matrix: - env: TOX_SUFFIX="flakes" python: "pypy3.5-5.9.0" - env: TOX_SUFFIX="aiohttp" - python: 2.7 + python: "2.7" - env: TOX_SUFFIX="aiohttp" - python: pypy + python: "pypy2.7-7.0" python: - 2.7 - 3.5 diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml index e132e5a..e319dc8 100644 --- a/tests/fixtures/migration/new_cassette.yaml +++ b/tests/fixtures/migration/new_cassette.yaml @@ -9,7 +9,7 @@ interactions: method: GET uri: http://httpbin.org/ip response: - body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} + body: {string: "{\n \"origin\": \"217.122.164.194\"\n}"} headers: access-control-allow-origin: ['*'] content-type: [application/json] diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index bd35558..c6bca7f 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -1,15 +1,46 @@ import pytest +import os + boto3 = pytest.importorskip("boto3") import boto3 # NOQA +import botocore # NOQA import vcr # NOQA -bucket = 'boto3-demo-1337' # a bucket you can access -key = 'test/my_test.txt' # key with r+w access -content = 'hello world i am a string' # content to put in the test file +ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_session_token=None, + region_name=os.environ['AWS_DEFAULT_REGION'], + # botocore_session=None, + # profile_name=None +) + +IAM_CLIENT = ses.client('iam') + +try: + from botocore import awsrequest # NOQA + + botocore_awsrequest = True +except ImportError: + botocore_awsrequest = False -def test_boto_stubs(tmpdir): +# skip tests if boto does not use vendored requests anymore +# https://github.com/boto/botocore/pull/1495 +boto3_skip_vendored_requests = pytest.mark.skipif( + botocore_awsrequest, + reason='botocore version {ver} does not use vendored requests anymore.'.format( + ver=botocore.__version__)) + +boto3_skip_awsrequest = pytest.mark.skipif( + not botocore_awsrequest, + reason='botocore version {ver} still uses vendored requests.'.format( + ver=botocore.__version__)) + + +@boto3_skip_vendored_requests +def test_boto_vendored_stubs(tmpdir): with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): # Perform the imports within the patched context so that # HTTPConnection, VerifiedHTTPSConnection refers to the patched version. @@ -23,45 +54,59 @@ def test_boto_stubs(tmpdir): VerifiedHTTPSConnection('hostname.does.not.matter') +@boto3_skip_awsrequest +def test_boto3_awsrequest_stubs(tmpdir): + with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): + from botocore.awsrequest import AWSHTTPConnection, AWSHTTPSConnection + from vcr.stubs.boto3_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + assert issubclass(VCRRequestsHTTPConnection, AWSHTTPConnection) + assert issubclass(VCRRequestsHTTPSConnection, AWSHTTPSConnection) + AWSHTTPConnection('hostname.does.not.matter') + AWSHTTPSConnection('hostname.does.not.matter') + + def test_boto3_without_vcr(): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) + username = 'user' + response = IAM_CLIENT.get_user(UserName=username) - # retrieve content to check it - o = s3_resource.Object(bucket, key).get() - - # decode for python3 - assert content == o['Body'].read().decode('utf-8') + assert response['User']['UserName'] == username def test_boto_medium_difficulty(tmpdir): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) + username = 'user' + with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))): - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + response = IAM_CLIENT.get_user(UserName=username) + assert response['User']['UserName'] == username with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass: - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + response = IAM_CLIENT.get_user(UserName=username) + assert response['User']['UserName'] == username assert cass.all_played def test_boto_hardcore_mode(tmpdir): + username = 'user' with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + region_name=os.environ['AWS_DEFAULT_REGION'], + ) + + iam_client = ses.client('iam') + response = iam_client.get_user(UserName=username) + assert response['User']['UserName'] == username with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass: - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_session_token=None, + region_name=os.environ['AWS_DEFAULT_REGION'], + ) + + iam_client = ses.client('iam') + response = iam_client.get_user(UserName=username) + assert response['User']['UserName'] == username assert cass.all_played diff --git a/tox.ini b/tox.ini index eb62410..5f0afd4 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps = requests: requests>=2.22.0 httplib2: httplib2 urllib3121: urllib3==1.21.1 + urllib3: urllib3 {py27,py35,py36,pypy}-tornado4: tornado>=4,<5 {py27,py35,py36,pypy}-tornado4: pytest-tornado {py27,py35,py36}-tornado4: pycurl @@ -29,6 +30,9 @@ deps = aiohttp: aiohttp aiohttp: pytest-asyncio aiohttp: pytest-aiohttp - +passenv = + AWS_ACCESS_KEY_ID + AWS_DEFAULT_REGION + AWS_SECRET_ACCESS_KEY [flake8] max_line_length = 110 diff --git a/vcr/cassette.py b/vcr/cassette.py index 7ba092c..70ed20e 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -190,6 +190,7 @@ class Cassette(object): self._serializer = serializer or yamlserializer self._match_on = match_on self._before_record_request = before_record_request or (lambda x: x) + log.info(self._before_record_request) self._before_record_response = before_record_response or (lambda x: x) self.inject = inject self.record_mode = record_mode @@ -225,6 +226,7 @@ class Cassette(object): def append(self, request, response): """Add a request, response pair to this cassette""" + log.info("Appending request %s and response %s", request, response) request = self._before_record_request(request) if not request: return diff --git a/vcr/patch.py b/vcr/patch.py index ac8ef29..b818c16 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -6,21 +6,29 @@ from .compat import contextlib, mock from .stubs import VCRHTTPConnection, VCRHTTPSConnection from six.moves import http_client as httplib +import logging +log = logging.getLogger(__name__) # Save some of the original types for the purposes of unpatching _HTTPConnection = httplib.HTTPConnection _HTTPSConnection = httplib.HTTPSConnection - # Try to save the original types for boto3 try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool -except ImportError: # pragma: no cover - pass + from botocore.awsrequest import AWSHTTPSConnection, AWSHTTPConnection +except ImportError: + try: + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + _cpoolBoto3HTTPConnection = cpool.HTTPConnection + _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection else: - _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection - _cpoolBoto3HTTPConnection = cpool.HTTPConnection - _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection + _Boto3VerifiedHTTPSConnection = AWSHTTPSConnection + _cpoolBoto3HTTPConnection = AWSHTTPConnection + _cpoolBoto3HTTPSConnection = AWSHTTPSConnection cpool = None # Try to save the original types for urllib3 @@ -44,7 +52,6 @@ else: _cpoolHTTPConnection = cpool.HTTPConnection _cpoolHTTPSConnection = cpool.HTTPSConnection - # Try to save the original types for httplib2 try: import httplib2 @@ -55,7 +62,6 @@ else: _HTTPSConnectionWithTimeout = httplib2.HTTPSConnectionWithTimeout _SCHEME_TO_CONNECTION = httplib2.SCHEME_TO_CONNECTION - # Try to save the original types for boto try: import boto.https_connection @@ -64,7 +70,6 @@ except ImportError: # pragma: no cover else: _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection - # Try to save the original types for Tornado try: import tornado.simple_httpclient @@ -74,7 +79,6 @@ else: _SimpleAsyncHTTPClient_fetch_impl = \ tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch_impl - try: import tornado.curl_httpclient except ImportError: # pragma: no cover @@ -99,6 +103,7 @@ class CassettePatcherBuilder(object): return self._build_patchers_from_mock_triples( function(self, *args, **kwargs) ) + return wrapped def __init__(self, cassette): @@ -184,13 +189,26 @@ class CassettePatcherBuilder(object): return () return self._urllib3_patchers(cpool, requests_stubs) + @_build_patchers_from_mock_triples_decorator def _boto3(self): + try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + # botocore using awsrequest + import botocore.awsrequest as cpool except ImportError: # pragma: no cover - return () - from .stubs import boto3_stubs - return self._urllib3_patchers(cpool, boto3_stubs) + try: + # botocore using vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs import boto3_stubs + yield self._urllib3_patchers(cpool, boto3_stubs) + else: + from .stubs import boto3_stubs + log.debug("Patching boto3 cpool with %s", cpool) + yield cpool.AWSHTTPConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPConnection + yield cpool.AWSHTTPSConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPSConnection def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @@ -407,22 +425,36 @@ def reset_patchers(): yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _cpoolHTTPSConnection) try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + # unpatch botocore with awsrequest + import botocore.awsrequest as cpool except ImportError: # pragma: no cover - pass + try: + # unpatch botocore with vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + # unpatch requests v1.x + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection) + yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection) + # unpatch requests v2.x + if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + _cpoolBoto3HTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + _cpoolBoto3HTTPSConnection) + + if hasattr(cpool, 'HTTPSConnection'): + yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection) else: - # unpatch requests v1.x - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection) - yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection) - # unpatch requests v2.x - if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + if hasattr(cpool.AWSHTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.AWSHTTPConnectionPool, 'ConnectionCls', _cpoolBoto3HTTPConnection) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + yield mock.patch.object(cpool.AWSHTTPSConnectionPool, 'ConnectionCls', _cpoolBoto3HTTPSConnection) - if hasattr(cpool, 'HTTPSConnection'): - yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection) + if hasattr(cpool, 'AWSHTTPSConnection'): + yield mock.patch.object(cpool, 'AWSHTTPSConnection', _cpoolBoto3HTTPSConnection) try: import httplib2 as cpool diff --git a/vcr/request.py b/vcr/request.py index b4b3f71..bf22f00 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -2,6 +2,9 @@ import warnings from six import BytesIO, text_type from six.moves.urllib.parse import urlparse, parse_qsl from .util import CaseInsensitiveDict +import logging + +log = logging.getLogger(__name__) class Request(object): @@ -18,6 +21,7 @@ class Request(object): else: self.body = body self.headers = headers + log.debug("Invoking Request %s", self.uri) @property def headers(self): diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 118bc22..9aac3d0 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -170,6 +170,7 @@ class VCRConnection(object): self._port_postfix(), url, ) + log.debug("Absolute URI: %s", uri) return uri def _url(self, uri): diff --git a/vcr/stubs/boto3_stubs.py b/vcr/stubs/boto3_stubs.py index 93a7bdb..80a03f9 100644 --- a/vcr/stubs/boto3_stubs.py +++ b/vcr/stubs/boto3_stubs.py @@ -1,11 +1,21 @@ -'''Stubs for boto3''' +"""Stubs for boto3""" + +try: + # boto using awsrequest + from botocore.awsrequest import AWSHTTPConnection as HTTPConnection + from botocore.awsrequest import AWSHTTPSConnection as VerifiedHTTPSConnection + +except ImportError: # pragma: nocover + # boto using vendored requests + # urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes + # you're using. It includes some polyfills for newer features missing in older pythons. + try: + from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection + except ImportError: # pragma: nocover + from requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection -from botocore.vendored.requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection from ..stubs import VCRHTTPConnection, VCRHTTPSConnection -# urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes -# you're using. It includes some polyfills for newer features missing in older pythons. - class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): _baseclass = HTTPConnection