From 347026f42c9fa9251fa89c347afce40f996892fc Mon Sep 17 00:00:00 2001 From: Jan Gazda <1oglop1@gmail.com> Date: Mon, 13 May 2019 11:33:19 +0200 Subject: [PATCH 1/6] Fix #382 - boto3 compatibility * Add support for PyYaml5.1 * Unpin requests in Tox * Unpin urllib3 in Tox * Unpin Flask in Tox * Add env vars to Tox for boto3 tests --- .travis.yml | 5 +- tests/fixtures/migration/new_cassette.yaml | 2 +- tests/integration/test_boto3.py | 105 +++++++++++++++------ tox.ini | 6 +- vcr/cassette.py | 2 + vcr/patch.py | 84 ++++++++++++----- vcr/request.py | 4 + vcr/stubs/__init__.py | 1 + vcr/stubs/boto3_stubs.py | 20 +++- 9 files changed, 164 insertions(+), 65 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4988390..e26e4e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: - TOX_SUFFIX="httplib2" - TOX_SUFFIX="boto3" - TOX_SUFFIX="urllib3121" + - TOX_SUFFIX="urllib3" - TOX_SUFFIX="tornado4" - TOX_SUFFIX="aiohttp" matrix: @@ -45,9 +46,9 @@ matrix: - env: TOX_SUFFIX="flakes" python: "pypy3.5-5.9.0" - env: TOX_SUFFIX="aiohttp" - python: 2.7 + python: "2.7" - env: TOX_SUFFIX="aiohttp" - python: pypy + python: "pypy2.7-7.0" python: - 2.7 - 3.5 diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml index e132e5a..e319dc8 100644 --- a/tests/fixtures/migration/new_cassette.yaml +++ b/tests/fixtures/migration/new_cassette.yaml @@ -9,7 +9,7 @@ interactions: method: GET uri: http://httpbin.org/ip response: - body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} + body: {string: "{\n \"origin\": \"217.122.164.194\"\n}"} headers: access-control-allow-origin: ['*'] content-type: [application/json] diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index bd35558..c6bca7f 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -1,15 +1,46 @@ import pytest +import os + boto3 = pytest.importorskip("boto3") import boto3 # NOQA +import botocore # NOQA import vcr # NOQA -bucket = 'boto3-demo-1337' # a bucket you can access -key = 'test/my_test.txt' # key with r+w access -content = 'hello world i am a string' # content to put in the test file +ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_session_token=None, + region_name=os.environ['AWS_DEFAULT_REGION'], + # botocore_session=None, + # profile_name=None +) + +IAM_CLIENT = ses.client('iam') + +try: + from botocore import awsrequest # NOQA + + botocore_awsrequest = True +except ImportError: + botocore_awsrequest = False -def test_boto_stubs(tmpdir): +# skip tests if boto does not use vendored requests anymore +# https://github.com/boto/botocore/pull/1495 +boto3_skip_vendored_requests = pytest.mark.skipif( + botocore_awsrequest, + reason='botocore version {ver} does not use vendored requests anymore.'.format( + ver=botocore.__version__)) + +boto3_skip_awsrequest = pytest.mark.skipif( + not botocore_awsrequest, + reason='botocore version {ver} still uses vendored requests.'.format( + ver=botocore.__version__)) + + +@boto3_skip_vendored_requests +def test_boto_vendored_stubs(tmpdir): with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): # Perform the imports within the patched context so that # HTTPConnection, VerifiedHTTPSConnection refers to the patched version. @@ -23,45 +54,59 @@ def test_boto_stubs(tmpdir): VerifiedHTTPSConnection('hostname.does.not.matter') +@boto3_skip_awsrequest +def test_boto3_awsrequest_stubs(tmpdir): + with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): + from botocore.awsrequest import AWSHTTPConnection, AWSHTTPSConnection + from vcr.stubs.boto3_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + assert issubclass(VCRRequestsHTTPConnection, AWSHTTPConnection) + assert issubclass(VCRRequestsHTTPSConnection, AWSHTTPSConnection) + AWSHTTPConnection('hostname.does.not.matter') + AWSHTTPSConnection('hostname.does.not.matter') + + def test_boto3_without_vcr(): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) + username = 'user' + response = IAM_CLIENT.get_user(UserName=username) - # retrieve content to check it - o = s3_resource.Object(bucket, key).get() - - # decode for python3 - assert content == o['Body'].read().decode('utf-8') + assert response['User']['UserName'] == username def test_boto_medium_difficulty(tmpdir): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) + username = 'user' + with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))): - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + response = IAM_CLIENT.get_user(UserName=username) + assert response['User']['UserName'] == username with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass: - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + response = IAM_CLIENT.get_user(UserName=username) + assert response['User']['UserName'] == username assert cass.all_played def test_boto_hardcore_mode(tmpdir): + username = 'user' with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + region_name=os.environ['AWS_DEFAULT_REGION'], + ) + + iam_client = ses.client('iam') + response = iam_client.get_user(UserName=username) + assert response['User']['UserName'] == username with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass: - s3_resource = boto3.resource('s3') - b = s3_resource.Bucket(bucket) - b.put_object(Key=key, Body=content) - o = s3_resource.Object(bucket, key).get() - assert content == o['Body'].read().decode('utf-8') + ses = boto3.Session( + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_session_token=None, + region_name=os.environ['AWS_DEFAULT_REGION'], + ) + + iam_client = ses.client('iam') + response = iam_client.get_user(UserName=username) + assert response['User']['UserName'] == username assert cass.all_played diff --git a/tox.ini b/tox.ini index eb62410..5f0afd4 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps = requests: requests>=2.22.0 httplib2: httplib2 urllib3121: urllib3==1.21.1 + urllib3: urllib3 {py27,py35,py36,pypy}-tornado4: tornado>=4,<5 {py27,py35,py36,pypy}-tornado4: pytest-tornado {py27,py35,py36}-tornado4: pycurl @@ -29,6 +30,9 @@ deps = aiohttp: aiohttp aiohttp: pytest-asyncio aiohttp: pytest-aiohttp - +passenv = + AWS_ACCESS_KEY_ID + AWS_DEFAULT_REGION + AWS_SECRET_ACCESS_KEY [flake8] max_line_length = 110 diff --git a/vcr/cassette.py b/vcr/cassette.py index 7ba092c..70ed20e 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -190,6 +190,7 @@ class Cassette(object): self._serializer = serializer or yamlserializer self._match_on = match_on self._before_record_request = before_record_request or (lambda x: x) + log.info(self._before_record_request) self._before_record_response = before_record_response or (lambda x: x) self.inject = inject self.record_mode = record_mode @@ -225,6 +226,7 @@ class Cassette(object): def append(self, request, response): """Add a request, response pair to this cassette""" + log.info("Appending request %s and response %s", request, response) request = self._before_record_request(request) if not request: return diff --git a/vcr/patch.py b/vcr/patch.py index ac8ef29..b818c16 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -6,21 +6,29 @@ from .compat import contextlib, mock from .stubs import VCRHTTPConnection, VCRHTTPSConnection from six.moves import http_client as httplib +import logging +log = logging.getLogger(__name__) # Save some of the original types for the purposes of unpatching _HTTPConnection = httplib.HTTPConnection _HTTPSConnection = httplib.HTTPSConnection - # Try to save the original types for boto3 try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool -except ImportError: # pragma: no cover - pass + from botocore.awsrequest import AWSHTTPSConnection, AWSHTTPConnection +except ImportError: + try: + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + _cpoolBoto3HTTPConnection = cpool.HTTPConnection + _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection else: - _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection - _cpoolBoto3HTTPConnection = cpool.HTTPConnection - _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection + _Boto3VerifiedHTTPSConnection = AWSHTTPSConnection + _cpoolBoto3HTTPConnection = AWSHTTPConnection + _cpoolBoto3HTTPSConnection = AWSHTTPSConnection cpool = None # Try to save the original types for urllib3 @@ -44,7 +52,6 @@ else: _cpoolHTTPConnection = cpool.HTTPConnection _cpoolHTTPSConnection = cpool.HTTPSConnection - # Try to save the original types for httplib2 try: import httplib2 @@ -55,7 +62,6 @@ else: _HTTPSConnectionWithTimeout = httplib2.HTTPSConnectionWithTimeout _SCHEME_TO_CONNECTION = httplib2.SCHEME_TO_CONNECTION - # Try to save the original types for boto try: import boto.https_connection @@ -64,7 +70,6 @@ except ImportError: # pragma: no cover else: _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection - # Try to save the original types for Tornado try: import tornado.simple_httpclient @@ -74,7 +79,6 @@ else: _SimpleAsyncHTTPClient_fetch_impl = \ tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch_impl - try: import tornado.curl_httpclient except ImportError: # pragma: no cover @@ -99,6 +103,7 @@ class CassettePatcherBuilder(object): return self._build_patchers_from_mock_triples( function(self, *args, **kwargs) ) + return wrapped def __init__(self, cassette): @@ -184,13 +189,26 @@ class CassettePatcherBuilder(object): return () return self._urllib3_patchers(cpool, requests_stubs) + @_build_patchers_from_mock_triples_decorator def _boto3(self): + try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + # botocore using awsrequest + import botocore.awsrequest as cpool except ImportError: # pragma: no cover - return () - from .stubs import boto3_stubs - return self._urllib3_patchers(cpool, boto3_stubs) + try: + # botocore using vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs import boto3_stubs + yield self._urllib3_patchers(cpool, boto3_stubs) + else: + from .stubs import boto3_stubs + log.debug("Patching boto3 cpool with %s", cpool) + yield cpool.AWSHTTPConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPConnection + yield cpool.AWSHTTPSConnectionPool, 'ConnectionCls', boto3_stubs.VCRRequestsHTTPSConnection def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @@ -407,22 +425,36 @@ def reset_patchers(): yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _cpoolHTTPSConnection) try: - import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + # unpatch botocore with awsrequest + import botocore.awsrequest as cpool except ImportError: # pragma: no cover - pass + try: + # unpatch botocore with vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + # unpatch requests v1.x + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection) + yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection) + # unpatch requests v2.x + if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + _cpoolBoto3HTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + _cpoolBoto3HTTPSConnection) + + if hasattr(cpool, 'HTTPSConnection'): + yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection) else: - # unpatch requests v1.x - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _Boto3VerifiedHTTPSConnection) - yield mock.patch.object(cpool, 'HTTPConnection', _cpoolBoto3HTTPConnection) - # unpatch requests v2.x - if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + if hasattr(cpool.AWSHTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.AWSHTTPConnectionPool, 'ConnectionCls', _cpoolBoto3HTTPConnection) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + yield mock.patch.object(cpool.AWSHTTPSConnectionPool, 'ConnectionCls', _cpoolBoto3HTTPSConnection) - if hasattr(cpool, 'HTTPSConnection'): - yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolBoto3HTTPSConnection) + if hasattr(cpool, 'AWSHTTPSConnection'): + yield mock.patch.object(cpool, 'AWSHTTPSConnection', _cpoolBoto3HTTPSConnection) try: import httplib2 as cpool diff --git a/vcr/request.py b/vcr/request.py index b4b3f71..bf22f00 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -2,6 +2,9 @@ import warnings from six import BytesIO, text_type from six.moves.urllib.parse import urlparse, parse_qsl from .util import CaseInsensitiveDict +import logging + +log = logging.getLogger(__name__) class Request(object): @@ -18,6 +21,7 @@ class Request(object): else: self.body = body self.headers = headers + log.debug("Invoking Request %s", self.uri) @property def headers(self): diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 118bc22..9aac3d0 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -170,6 +170,7 @@ class VCRConnection(object): self._port_postfix(), url, ) + log.debug("Absolute URI: %s", uri) return uri def _url(self, uri): diff --git a/vcr/stubs/boto3_stubs.py b/vcr/stubs/boto3_stubs.py index 93a7bdb..80a03f9 100644 --- a/vcr/stubs/boto3_stubs.py +++ b/vcr/stubs/boto3_stubs.py @@ -1,11 +1,21 @@ -'''Stubs for boto3''' +"""Stubs for boto3""" + +try: + # boto using awsrequest + from botocore.awsrequest import AWSHTTPConnection as HTTPConnection + from botocore.awsrequest import AWSHTTPSConnection as VerifiedHTTPSConnection + +except ImportError: # pragma: nocover + # boto using vendored requests + # urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes + # you're using. It includes some polyfills for newer features missing in older pythons. + try: + from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection + except ImportError: # pragma: nocover + from requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection -from botocore.vendored.requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection from ..stubs import VCRHTTPConnection, VCRHTTPSConnection -# urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes -# you're using. It includes some polyfills for newer features missing in older pythons. - class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): _baseclass = HTTPConnection From 4daafcc68702226cbbff1f2b47c9a029f09e1717 Mon Sep 17 00:00:00 2001 From: Jan Gazda <1oglop1@gmail.com> Date: Mon, 1 Jul 2019 08:38:08 +0200 Subject: [PATCH 2/6] Rename testing user --- tests/integration/test_boto3.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index c6bca7f..b1e9dd2 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -17,6 +17,7 @@ ses = boto3.Session( ) IAM_CLIENT = ses.client('iam') +IAM_USER_NAME = 'vcrpy' try: from botocore import awsrequest # NOQA @@ -66,27 +67,24 @@ def test_boto3_awsrequest_stubs(tmpdir): def test_boto3_without_vcr(): - username = 'user' - response = IAM_CLIENT.get_user(UserName=username) + response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) - assert response['User']['UserName'] == username + assert response['User']['UserName'] == IAM_USER_NAME def test_boto_medium_difficulty(tmpdir): - username = 'user' with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))): - response = IAM_CLIENT.get_user(UserName=username) - assert response['User']['UserName'] == username + response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) + assert response['User']['UserName'] == IAM_USER_NAME with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass: - response = IAM_CLIENT.get_user(UserName=username) - assert response['User']['UserName'] == username + response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) + assert response['User']['UserName'] == IAM_USER_NAME assert cass.all_played def test_boto_hardcore_mode(tmpdir): - username = 'user' with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): ses = boto3.Session( aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], @@ -95,8 +93,8 @@ def test_boto_hardcore_mode(tmpdir): ) iam_client = ses.client('iam') - response = iam_client.get_user(UserName=username) - assert response['User']['UserName'] == username + response = iam_client.get_user(UserName=IAM_USER_NAME) + assert response['User']['UserName'] == IAM_USER_NAME with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass: ses = boto3.Session( @@ -107,6 +105,6 @@ def test_boto_hardcore_mode(tmpdir): ) iam_client = ses.client('iam') - response = iam_client.get_user(UserName=username) - assert response['User']['UserName'] == username + response = iam_client.get_user(UserName=IAM_USER_NAME) + assert response['User']['UserName'] == IAM_USER_NAME assert cass.all_played From 14d1454bbff8ca35503705ea48bb76e2bce465fa Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Fri, 5 Jul 2019 23:27:13 +0200 Subject: [PATCH 3/6] test with latest urllib3 --- .travis.yml | 7 +++---- tox.ini | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e26e4e8..50848d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ env: - TOX_SUFFIX="requests" - TOX_SUFFIX="httplib2" - TOX_SUFFIX="boto3" - - TOX_SUFFIX="urllib3121" - TOX_SUFFIX="urllib3" - TOX_SUFFIX="tornado4" - TOX_SUFFIX="aiohttp" @@ -24,7 +23,7 @@ matrix: - env: TOX_SUFFIX="httplib2" python: 3.7 dist: xenial - - env: TOX_SUFFIX="urllib3121" + - env: TOX_SUFFIX="urllib3" python: 3.7 dist: xenial - env: TOX_SUFFIX="tornado4" @@ -46,9 +45,9 @@ matrix: - env: TOX_SUFFIX="flakes" python: "pypy3.5-5.9.0" - env: TOX_SUFFIX="aiohttp" - python: "2.7" + python: 2.7 - env: TOX_SUFFIX="aiohttp" - python: "pypy2.7-7.0" + python: pypy python: - 2.7 - 3.5 diff --git a/tox.ini b/tox.ini index 5f0afd4..e4a4274 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3121,tornado4,boto3},{py35,py36,py37}-{aiohttp} +envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp} [testenv:flakes] skipsdist = True @@ -21,12 +21,12 @@ deps = ipaddress requests: requests>=2.22.0 httplib2: httplib2 - urllib3121: urllib3==1.21.1 urllib3: urllib3 {py27,py35,py36,pypy}-tornado4: tornado>=4,<5 {py27,py35,py36,pypy}-tornado4: pytest-tornado {py27,py35,py36}-tornado4: pycurl boto3: boto3 + boto3: urllib3 aiohttp: aiohttp aiohttp: pytest-asyncio aiohttp: pytest-aiohttp From 5e76e4733db22c30aca93ef4e7c1b0c4bbfd0b16 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Sun, 7 Jul 2019 20:15:09 +0200 Subject: [PATCH 4/6] fix real connection on boto3 stub, adding not set parameters manually Set `assert_hostname` and `cert_reqs` real connection attributes with default values. --- tests/integration/test_boto3.py | 18 +++++++++--------- vcr/stubs/boto3_stubs.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index b1e9dd2..1e418e7 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -8,10 +8,10 @@ import botocore # NOQA import vcr # NOQA ses = boto3.Session( - aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', "default"), + aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY', "default"), aws_session_token=None, - region_name=os.environ['AWS_DEFAULT_REGION'], + region_name=os.environ.get('AWS_DEFAULT_REGION', "default"), # botocore_session=None, # profile_name=None ) @@ -87,9 +87,9 @@ def test_boto_medium_difficulty(tmpdir): def test_boto_hardcore_mode(tmpdir): with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): ses = boto3.Session( - aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], - region_name=os.environ['AWS_DEFAULT_REGION'], + aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'), + region_name=os.environ.get('AWS_DEFAULT_REGION'), ) iam_client = ses.client('iam') @@ -98,10 +98,10 @@ def test_boto_hardcore_mode(tmpdir): with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass: ses = boto3.Session( - aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'), aws_session_token=None, - region_name=os.environ['AWS_DEFAULT_REGION'], + region_name=os.environ.get('AWS_DEFAULT_REGION'), ) iam_client = ses.client('iam') diff --git a/vcr/stubs/boto3_stubs.py b/vcr/stubs/boto3_stubs.py index 80a03f9..d83fac2 100644 --- a/vcr/stubs/boto3_stubs.py +++ b/vcr/stubs/boto3_stubs.py @@ -1,4 +1,5 @@ """Stubs for boto3""" +import six try: # boto using awsrequest @@ -23,3 +24,21 @@ class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): _baseclass = VerifiedHTTPSConnection + + def __init__(self, *args, **kwargs): + if six.PY3: + kwargs.pop('strict', None) # apparently this is gone in py3 + + # need to temporarily reset here because the real connection + # inherits from the thing that we are mocking out. Take out + # the reset if you want to see what I mean :) + from vcr.patch import force_reset + with force_reset(): + self.real_connection = self._baseclass(*args, **kwargs) + # Make sure to set those attributes as it seems `AWSHTTPConnection` does not + # set them, making the connection to fail ! + self.real_connection.assert_hostname = kwargs.get("assert_hostname", False) + self.real_connection.cert_reqs = kwargs.get("cert_reqs", 'CERT_NONE') + + self._sock = None + From 6ca7cf3cc6b879b46f2652dcdbb6c82cf22cc0e7 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Sat, 13 Jul 2019 13:08:03 +0200 Subject: [PATCH 5/6] refactor boto3 integration test using fixtures instead of global variables --- tests/integration/test_boto3.py | 73 +++++++++++++++++---------------- vcr/stubs/boto3_stubs.py | 1 - 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index 1e418e7..74af71e 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -7,18 +7,6 @@ import boto3 # NOQA import botocore # NOQA import vcr # NOQA -ses = boto3.Session( - aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', "default"), - aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY', "default"), - aws_session_token=None, - region_name=os.environ.get('AWS_DEFAULT_REGION', "default"), - # botocore_session=None, - # profile_name=None -) - -IAM_CLIENT = ses.client('iam') -IAM_USER_NAME = 'vcrpy' - try: from botocore import awsrequest # NOQA @@ -26,7 +14,6 @@ try: except ImportError: botocore_awsrequest = False - # skip tests if boto does not use vendored requests anymore # https://github.com/boto/botocore/pull/1495 boto3_skip_vendored_requests = pytest.mark.skipif( @@ -40,6 +27,33 @@ boto3_skip_awsrequest = pytest.mark.skipif( ver=botocore.__version__)) +IAM_USER_NAME = "vcrpy" + + +@pytest.fixture +def iam_client(): + def _iam_client(boto3_session=None): + if boto3_session is None: + boto3_session = boto3.Session( + aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', "default"), + aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY', "default"), + aws_session_token=None, + region_name=os.environ.get('AWS_DEFAULT_REGION', "default"), + ) + return boto3_session.client('iam') + return _iam_client + + +@pytest.fixture +def get_user(iam_client): + def _get_user(client=None, user_name=IAM_USER_NAME): + if client is None: + # Default client set with fixture `iam_client` + client = iam_client() + return client.get_user(UserName=user_name) + return _get_user + + @boto3_skip_vendored_requests def test_boto_vendored_stubs(tmpdir): with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): @@ -55,45 +69,32 @@ def test_boto_vendored_stubs(tmpdir): VerifiedHTTPSConnection('hostname.does.not.matter') -@boto3_skip_awsrequest -def test_boto3_awsrequest_stubs(tmpdir): - with vcr.use_cassette(str(tmpdir.join('boto3-stubs.yml'))): - from botocore.awsrequest import AWSHTTPConnection, AWSHTTPSConnection - from vcr.stubs.boto3_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection - assert issubclass(VCRRequestsHTTPConnection, AWSHTTPConnection) - assert issubclass(VCRRequestsHTTPSConnection, AWSHTTPSConnection) - AWSHTTPConnection('hostname.does.not.matter') - AWSHTTPSConnection('hostname.does.not.matter') - - -def test_boto3_without_vcr(): - response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) - +def test_boto3_without_vcr(get_user): + response = get_user() assert response['User']['UserName'] == IAM_USER_NAME -def test_boto_medium_difficulty(tmpdir): +def test_boto_medium_difficulty(tmpdir, get_user): with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))): - response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) + response = get_user() assert response['User']['UserName'] == IAM_USER_NAME with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))) as cass: - response = IAM_CLIENT.get_user(UserName=IAM_USER_NAME) + response = get_user() assert response['User']['UserName'] == IAM_USER_NAME assert cass.all_played -def test_boto_hardcore_mode(tmpdir): +def test_boto_hardcore_mode(tmpdir, iam_client, get_user): with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): ses = boto3.Session( aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'), region_name=os.environ.get('AWS_DEFAULT_REGION'), ) - - iam_client = ses.client('iam') - response = iam_client.get_user(UserName=IAM_USER_NAME) + client = iam_client(ses) + response = get_user(client=client) assert response['User']['UserName'] == IAM_USER_NAME with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))) as cass: @@ -104,7 +105,7 @@ def test_boto_hardcore_mode(tmpdir): region_name=os.environ.get('AWS_DEFAULT_REGION'), ) - iam_client = ses.client('iam') - response = iam_client.get_user(UserName=IAM_USER_NAME) + client = iam_client(ses) + response = get_user(client=client) assert response['User']['UserName'] == IAM_USER_NAME assert cass.all_played diff --git a/vcr/stubs/boto3_stubs.py b/vcr/stubs/boto3_stubs.py index d83fac2..73a455f 100644 --- a/vcr/stubs/boto3_stubs.py +++ b/vcr/stubs/boto3_stubs.py @@ -41,4 +41,3 @@ class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): self.real_connection.cert_reqs = kwargs.get("cert_reqs", 'CERT_NONE') self._sock = None - From 2d96d35621884c377e01e908359d787ac8f4cad6 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Sat, 13 Jul 2019 13:08:49 +0200 Subject: [PATCH 6/6] delete test_boto3_without_vcr test as it does not test vcr module --- tests/integration/test_boto3.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index 74af71e..22e014a 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -69,11 +69,6 @@ def test_boto_vendored_stubs(tmpdir): VerifiedHTTPSConnection('hostname.does.not.matter') -def test_boto3_without_vcr(get_user): - response = get_user() - assert response['User']['UserName'] == IAM_USER_NAME - - def test_boto_medium_difficulty(tmpdir, get_user): with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))):