From cb05f4163c45ac244b39ab29c178f4d088d46189 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Tue, 16 Sep 2014 23:32:28 -0700 Subject: [PATCH 01/29] Add use_cassette class so functinos that are decorated with use_cassette can be called multiple times. --- tests/unit/test_cassettes.py | 16 ++++++++++++++++ vcr/cassette.py | 32 ++++++++++++++++++++++++++++---- vcr/config.py | 2 +- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index b612072..1b310c4 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -68,6 +68,22 @@ def test_cassette_cant_read_same_request_twice(): a.play_response('foo') +@mock.patch('vcr.cassette.requests_match', return_value=True) +@mock.patch('vcr.cassette.load_cassette', lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) +@mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) +@mock.patch('vcr.stubs.VCRHTTPResponse') +def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): + from six.moves import http_client as httplib + @Cassette.use_cassette('test') + def decorated_function(): + conn = httplib.HTTPConnection("www.python.org") + conn.request("GET", "/index.html") + conn.getresponse() + + for i in range(2): + decorated_function() + + def test_cassette_not_all_played(): a = Cassette('test') a.append('foo', 'bar') diff --git a/vcr/cassette.py b/vcr/cassette.py index 36deed3..fb3c77c 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -1,12 +1,10 @@ '''The container for recorded requests and responses''' - +import functools try: from collections import Counter except ImportError: from .compat.counter import Counter -from contextdecorator import ContextDecorator - # Internal imports from .patch import install, reset from .persist import load_cassette, save_cassette @@ -16,7 +14,29 @@ from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError -class Cassette(ContextDecorator): +class use_cassette(object): + + def __init__(self, cls, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.cls = cls + + def __enter__(self): + self._cassette = self.cls.load(*self.args, **self.kwargs) + return self._cassette.__enter__() + + def __exit__(self, *args): + return self._cassette.__exit__(*args) + + def __call__(self, function): + @functools.wraps(function) + def wrapped(*args, **kwargs): + with self: + return function(*args, **kwargs) + return wrapped + + +class Cassette(object): '''A container for recorded requests and responses''' @classmethod @@ -26,6 +46,10 @@ class Cassette(ContextDecorator): new_cassette._load() return new_cassette + @classmethod + def use_cassette(cls, *args, **kwargs): + return use_cassette(cls, *args, **kwargs) + def __init__(self, path, serializer=yamlserializer, diff --git a/vcr/config.py b/vcr/config.py index 0a1615a..1b868cd 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -103,7 +103,7 @@ class VCR(object): ), } - return Cassette.load(path, **merged_config) + return Cassette.use_cassette(path, **merged_config) def register_serializer(self, name, serializer): self.serializers[name] = serializer From 0cfe63ef6e6fe2ccfc4100611e6f910919cf313f Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Tue, 16 Sep 2014 23:53:50 -0700 Subject: [PATCH 02/29] Bump version number for new use_cassette_decorator. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dc2f629..7d47d1f 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ class PyTest(TestCommand): setup( name='vcrpy', - version='1.0.3', + version='1.0.4', description=( "Automatically mock your HTTP interactions to simplify and " "speed up testing" From 366e2b75bbba851325e63499c9770ebcf8f3faf7 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 01:28:54 -0700 Subject: [PATCH 03/29] Add global toggle to use_cassette. --- tests/unit/test_cassettes.py | 26 ++++++++++++++++++++++++-- vcr/__init__.py | 4 ++++ vcr/cassette.py | 20 +++++++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 1b310c4..eb54909 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -1,8 +1,12 @@ +from six.moves import http_client as httplib + import pytest import yaml import mock + from vcr.cassette import Cassette from vcr.errors import UnhandledHTTPRequestError +from vcr import global_toggle def test_cassette_load(tmpdir): @@ -69,11 +73,11 @@ def test_cassette_cant_read_same_request_twice(): @mock.patch('vcr.cassette.requests_match', return_value=True) -@mock.patch('vcr.cassette.load_cassette', lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) +@mock.patch('vcr.cassette.load_cassette', + lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) @mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) @mock.patch('vcr.stubs.VCRHTTPResponse') def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): - from six.moves import http_client as httplib @Cassette.use_cassette('test') def decorated_function(): conn = httplib.HTTPConnection("www.python.org") @@ -96,3 +100,21 @@ def test_cassette_all_played(): a.append('foo', 'bar') a.play_response('foo') assert a.all_played + +@mock.patch('vcr.cassette.install') +@mock.patch('vcr.cassette.reset') +def test_global_toggle(mock_reset, mock_install): + @Cassette.use_cassette('test') + def function(): + pass + + global_toggle(enabled=False) + + function() + assert mock_install.call_count == 0 + assert mock_reset.call_count == 0 + + global_toggle(enabled=True) + function() + mock_install.assert_called_once_with(mock.ANY) + mock_reset.assert_called_once_with() diff --git a/vcr/__init__.py b/vcr/__init__.py index 60af690..ccf8993 100644 --- a/vcr/__init__.py +++ b/vcr/__init__.py @@ -1,5 +1,6 @@ import logging from .config import VCR +from . import cassette # Set default logging handler to avoid "No handler found" warnings. try: # Python 2.7+ @@ -9,6 +10,9 @@ except ImportError: def emit(self, record): pass +def global_toggle(enabled=True): + cassette.use_cassette._enabled = enabled + logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/vcr/cassette.py b/vcr/cassette.py index fb3c77c..64a927b 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -14,15 +14,33 @@ from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError +class NullContextDecorator(object): + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + pass + + def __call__(self, function): + return function + + class use_cassette(object): + _enabled = True + def __init__(self, cls, *args, **kwargs): self.args = args self.kwargs = kwargs self.cls = cls def __enter__(self): - self._cassette = self.cls.load(*self.args, **self.kwargs) + self._cassette = self.cls.load(*self.args, **self.kwargs) if self._enabled \ + else NullContextDecorator() return self._cassette.__enter__() def __exit__(self, *args): From 9a4f5f23a4c690dc8954ec6a70ef858e08dfa46e Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 04:10:05 -0700 Subject: [PATCH 04/29] Add before_record_response to Cassette and VCR. --- .gitignore | 2 ++ tests/unit/test_cassettes.py | 10 ++++++++++ vcr/cassette.py | 4 ++++ vcr/config.py | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/.gitignore b/.gitignore index d0a4d9e..7805664 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ dist/ .coverage *.egg-info/ pytestdebug.log + +fixtures/ \ No newline at end of file diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index eb54909..518e510 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -118,3 +118,13 @@ def test_global_toggle(mock_reset, mock_install): function() mock_install.assert_called_once_with(mock.ANY) mock_reset.assert_called_once_with() + + +def test_before_record_response(): + before_record_response = mock.Mock(return_value='mutated') + cassette = Cassette('test', before_record_response=before_record_response) + cassette.append('req', 'res') + + before_record_response.assert_called_once_with('res') + assert cassette.responses[0] == 'mutated' + diff --git a/vcr/cassette.py b/vcr/cassette.py index 64a927b..bdbfeae 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -76,6 +76,7 @@ class Cassette(object): filter_headers=(), filter_query_parameters=(), before_record=None, + before_record_response=None, ignore_hosts=(), ignore_localhost=() ): @@ -85,6 +86,7 @@ class Cassette(object): self._filter_headers = filter_headers self._filter_query_parameters = filter_query_parameters self._before_record = before_record + self._before_record_response = before_record_response self._ignore_hosts = ignore_hosts if ignore_localhost: self._ignore_hosts = list(set( @@ -136,6 +138,8 @@ class Cassette(object): request = self._filter_request(request) if not request: return + if self._before_record_response: + response = self._before_record_response(response) self.data.append((request, response)) self.dirty = True diff --git a/vcr/config.py b/vcr/config.py index 1b868cd..9e5a80f 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -12,6 +12,7 @@ class VCR(object): filter_headers=[], filter_query_parameters=[], before_record=None, + before_record_response=None, match_on=[ 'method', 'scheme', @@ -46,6 +47,7 @@ class VCR(object): self.filter_headers = filter_headers self.filter_query_parameters = filter_query_parameters self.before_record = before_record + self.before_record_response = before_record_response self.ignore_hosts = ignore_hosts self.ignore_localhost = ignore_localhost @@ -95,6 +97,9 @@ class VCR(object): "before_record": kwargs.get( "before_record", self.before_record ), + "before_record_response": kwargs.get( + "before_record_response", self.before_record_response + ), "ignore_hosts": kwargs.get( 'ignore_hosts', self.ignore_hosts ), From 8e0142605610d07d9273497fd07cadbb1c99f63f Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 19:29:02 -0700 Subject: [PATCH 05/29] Change default paramters to VCR from lists to tuples. --- vcr/cassette.py | 3 +-- vcr/config.py | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index bdbfeae..f3934c3 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -78,8 +78,7 @@ class Cassette(object): before_record=None, before_record_response=None, ignore_hosts=(), - ignore_localhost=() - ): + ignore_localhost=()): self._path = path self._serializer = serializer self._match_on = match_on diff --git a/vcr/config.py b/vcr/config.py index 9e5a80f..9f7a318 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -9,19 +9,19 @@ class VCR(object): serializer='yaml', cassette_library_dir=None, record_mode="once", - filter_headers=[], - filter_query_parameters=[], + filter_headers=(), + filter_query_parameters=(), before_record=None, before_record_response=None, - match_on=[ + match_on=( 'method', 'scheme', 'host', 'port', 'path', 'query', - ], - ignore_hosts=[], + ), + ignore_hosts=(), ignore_localhost=False, ): self.serializer = serializer From a08c90c5d6fa08bc77acb0f3b5c92d3a89172a38 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 21:42:25 -0700 Subject: [PATCH 06/29] Revert "Add global toggle to use_cassette." This reverts commit 366e2b75bbba851325e63499c9770ebcf8f3faf7. Conflicts: tests/unit/test_cassettes.py --- tests/unit/test_cassettes.py | 27 ++------------------------- vcr/__init__.py | 4 ---- vcr/cassette.py | 20 +------------------- 3 files changed, 3 insertions(+), 48 deletions(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 518e510..1974e69 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -1,12 +1,8 @@ -from six.moves import http_client as httplib - import pytest import yaml import mock - from vcr.cassette import Cassette from vcr.errors import UnhandledHTTPRequestError -from vcr import global_toggle def test_cassette_load(tmpdir): @@ -73,11 +69,11 @@ def test_cassette_cant_read_same_request_twice(): @mock.patch('vcr.cassette.requests_match', return_value=True) -@mock.patch('vcr.cassette.load_cassette', - lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) +@mock.patch('vcr.cassette.load_cassette', lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) @mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) @mock.patch('vcr.stubs.VCRHTTPResponse') def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): + from six.moves import http_client as httplib @Cassette.use_cassette('test') def decorated_function(): conn = httplib.HTTPConnection("www.python.org") @@ -101,24 +97,6 @@ def test_cassette_all_played(): a.play_response('foo') assert a.all_played -@mock.patch('vcr.cassette.install') -@mock.patch('vcr.cassette.reset') -def test_global_toggle(mock_reset, mock_install): - @Cassette.use_cassette('test') - def function(): - pass - - global_toggle(enabled=False) - - function() - assert mock_install.call_count == 0 - assert mock_reset.call_count == 0 - - global_toggle(enabled=True) - function() - mock_install.assert_called_once_with(mock.ANY) - mock_reset.assert_called_once_with() - def test_before_record_response(): before_record_response = mock.Mock(return_value='mutated') @@ -127,4 +105,3 @@ def test_before_record_response(): before_record_response.assert_called_once_with('res') assert cassette.responses[0] == 'mutated' - diff --git a/vcr/__init__.py b/vcr/__init__.py index ccf8993..60af690 100644 --- a/vcr/__init__.py +++ b/vcr/__init__.py @@ -1,6 +1,5 @@ import logging from .config import VCR -from . import cassette # Set default logging handler to avoid "No handler found" warnings. try: # Python 2.7+ @@ -10,9 +9,6 @@ except ImportError: def emit(self, record): pass -def global_toggle(enabled=True): - cassette.use_cassette._enabled = enabled - logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/vcr/cassette.py b/vcr/cassette.py index f3934c3..b1864f2 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -14,33 +14,15 @@ from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError -class NullContextDecorator(object): - - def __init__(self, *args, **kwargs): - pass - - def __enter__(self, *args): - return self - - def __exit__(self, *args): - pass - - def __call__(self, function): - return function - - class use_cassette(object): - _enabled = True - def __init__(self, cls, *args, **kwargs): self.args = args self.kwargs = kwargs self.cls = cls def __enter__(self): - self._cassette = self.cls.load(*self.args, **self.kwargs) if self._enabled \ - else NullContextDecorator() + self._cassette = self.cls.load(*self.args, **self.kwargs) return self._cassette.__enter__() def __exit__(self, *args): From 8db46002a3a4627923dbbfaf7f6c95f42c424b3d Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 21:48:28 -0700 Subject: [PATCH 07/29] Fix failure in test_local_host resulting from attempting to add tuple to list. --- vcr/cassette.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index b1864f2..b39ae9c 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -71,7 +71,7 @@ class Cassette(object): self._ignore_hosts = ignore_hosts if ignore_localhost: self._ignore_hosts = list(set( - self._ignore_hosts + ['localhost', '0.0.0.0', '127.0.0.1'] + list(self._ignore_hosts) + ['localhost', '0.0.0.0', '127.0.0.1'] )) # self.data is the list of (req, resp) tuples From 472cc3bffea7dbff5d8879098185a2908de9aa42 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Wed, 17 Sep 2014 23:22:43 -0700 Subject: [PATCH 08/29] use_cassette -> CassetteContextDecorator --- setup.py | 2 +- tests/unit/test_cassettes.py | 6 ++++-- tests/unit/test_vcr.py | 16 ++++++++++++++++ vcr/cassette.py | 16 +++++++++++++--- vcr/config.py | 2 +- 5 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_vcr.py diff --git a/setup.py b/setup.py index 7d47d1f..0540e85 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( 'vcr.compat': 'vcr/compat', 'vcr.persisters': 'vcr/persisters', }, - install_requires=['PyYAML', 'contextdecorator', 'six'], + install_requires=['PyYAML', 'six'], license='MIT', tests_require=['pytest', 'mock', 'pytest-localserver'], cmdclass={'test': PyTest}, diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 1974e69..2589aa8 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -1,6 +1,9 @@ +from six.moves import http_client as httplib + import pytest import yaml import mock + from vcr.cassette import Cassette from vcr.errors import UnhandledHTTPRequestError @@ -73,8 +76,7 @@ def test_cassette_cant_read_same_request_twice(): @mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) @mock.patch('vcr.stubs.VCRHTTPResponse') def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): - from six.moves import http_client as httplib - @Cassette.use_cassette('test') + @Cassette.use('test') def decorated_function(): conn = httplib.HTTPConnection("www.python.org") conn.request("GET", "/index.html") diff --git a/tests/unit/test_vcr.py b/tests/unit/test_vcr.py new file mode 100644 index 0000000..aaae969 --- /dev/null +++ b/tests/unit/test_vcr.py @@ -0,0 +1,16 @@ +import mock + +from vcr import VCR + + + +def test_vcr_use_cassette(): + filter_headers = mock.Mock() + test_vcr = VCR(filter_headers=filter_headers) + with mock.patch('vcr.config.Cassette') as mock_cassette_class: + @test_vcr.use_cassette('test') + def function(): + pass + mock_cassette_class.call_count == 0 + function() + assert mock_cassette_class.use.call_args[1]['filter_headers'] is filter_headers diff --git a/vcr/cassette.py b/vcr/cassette.py index b39ae9c..4b10f18 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -14,7 +14,15 @@ from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError -class use_cassette(object): +class CassetteContextDecorator(object): + """Context manager/decorator that handles installing the cassette and + removing cassettes. + + This class defers the creation of a new cassette instance until the point at + which it is installed by context manager or decorator. The fact that a new + cassette is used with each application prevents the state of any cassette + from interfering with another. + """ def __init__(self, cls, *args, **kwargs): self.args = args @@ -47,8 +55,10 @@ class Cassette(object): return new_cassette @classmethod - def use_cassette(cls, *args, **kwargs): - return use_cassette(cls, *args, **kwargs) + def use(cls, *args, **kwargs): + return CassetteContextDecorator(cls, *args, **kwargs) + + use_cassette = use def __init__(self, path, diff --git a/vcr/config.py b/vcr/config.py index 9f7a318..0727372 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -108,7 +108,7 @@ class VCR(object): ), } - return Cassette.use_cassette(path, **merged_config) + return Cassette.use(path, **merged_config) def register_serializer(self, name, serializer): self.serializers[name] = serializer From 643a4c91ee5e4a5c743c28af5b75681b73c99308 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 02:52:44 -0700 Subject: [PATCH 09/29] Change use_cassette to pass a function to CassetteContextDecorator so that changes to the default settings on the vcr properly propogate. --- tests/unit/test_cassettes.py | 26 ++++++++++++++++++++++++++ tests/unit/test_vcr.py | 22 ++++++++++++++++++---- vcr/cassette.py | 20 +++++++++++++------- vcr/config.py | 9 ++++++--- 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 2589aa8..c0f415b 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -86,6 +86,32 @@ def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*arg decorated_function() +def test_arg_getter_functionality(): + arg_getter = mock.Mock(return_value=('test', {})) + context_decorator = Cassette.use_arg_getter(arg_getter) + + with context_decorator as cassette: + assert cassette._path == 'test' + + arg_getter.return_value = ('other', {}) + + with context_decorator as cassette: + assert cassette._path == 'other' + + arg_getter.return_value = ('', {'filter_headers': ('header_name',)}) + + @context_decorator + def function(): + pass + + with mock.patch.object(Cassette, '__init__', return_value=None) as cassette_init, \ + mock.patch.object(Cassette, '_load'), \ + mock.patch.object(Cassette, '__exit__'): + function() + cassette_init.assert_called_once_with(arg_getter.return_value[0], + **arg_getter.return_value[1]) + + def test_cassette_not_all_played(): a = Cassette('test') a.append('foo', 'bar') diff --git a/tests/unit/test_vcr.py b/tests/unit/test_vcr.py index aaae969..d460bc2 100644 --- a/tests/unit/test_vcr.py +++ b/tests/unit/test_vcr.py @@ -3,14 +3,28 @@ import mock from vcr import VCR - def test_vcr_use_cassette(): filter_headers = mock.Mock() test_vcr = VCR(filter_headers=filter_headers) - with mock.patch('vcr.config.Cassette') as mock_cassette_class: + with mock.patch('vcr.cassette.Cassette.__init__', return_value=None) as mock_cassette_init, \ + mock.patch('vcr.cassette.Cassette._load'), \ + mock.patch('vcr.cassette.Cassette.__exit__'): @test_vcr.use_cassette('test') def function(): pass - mock_cassette_class.call_count == 0 + assert mock_cassette_init.call_count == 0 function() - assert mock_cassette_class.use.call_args[1]['filter_headers'] is filter_headers + assert mock_cassette_init.call_args[1]['filter_headers'] is filter_headers + + # Make sure that calls to function now use cassettes with the + # new filter_header_settings + test_vcr.filter_headers = ('a',) + function() + assert mock_cassette_init.call_args[1]['filter_headers'] == test_vcr.filter_headers + + # Ensure that explicitly provided arguments still supercede + # those on the vcr. + new_filter_headers = mock.Mock() + + with test_vcr.use_cassette('test', filter_headers=new_filter_headers) as cassette: + assert cassette._filter_headers == new_filter_headers diff --git a/vcr/cassette.py b/vcr/cassette.py index 4b10f18..cb46e35 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -24,13 +24,17 @@ class CassetteContextDecorator(object): from interfering with another. """ - def __init__(self, cls, *args, **kwargs): - self.args = args - self.kwargs = kwargs + @classmethod + def from_args(cls, cassette_class, path, **kwargs): + return cls(cassette_class, lambda: (path, kwargs)) + + def __init__(self, cls, args_getter): self.cls = cls + self._args_getter = args_getter def __enter__(self): - self._cassette = self.cls.load(*self.args, **self.kwargs) + path, kwargs = self._args_getter() + self._cassette = self.cls.load(path, **kwargs) return self._cassette.__enter__() def __exit__(self, *args): @@ -55,10 +59,12 @@ class Cassette(object): return new_cassette @classmethod - def use(cls, *args, **kwargs): - return CassetteContextDecorator(cls, *args, **kwargs) + def use_arg_getter(cls, arg_getter): + return CassetteContextDecorator(cls, arg_getter) - use_cassette = use + @classmethod + def use(cls, *args, **kwargs): + return CassetteContextDecorator.from_args(cls, *args, **kwargs) def __init__(self, path, diff --git a/vcr/config.py b/vcr/config.py index 0727372..8308b48 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -1,3 +1,4 @@ +import functools import os from .cassette import Cassette from .serializers import yamlserializer, jsonserializer @@ -74,13 +75,16 @@ class VCR(object): return matchers def use_cassette(self, path, **kwargs): + args_getter = functools.partial(self.get_path_and_merged_config, path, **kwargs) + return Cassette.use_arg_getter(args_getter) + + def get_path_and_merged_config(self, path, **kwargs): serializer_name = kwargs.get('serializer', self.serializer) matcher_names = kwargs.get('match_on', self.match_on) cassette_library_dir = kwargs.get( 'cassette_library_dir', self.cassette_library_dir ) - if cassette_library_dir: path = os.path.join(cassette_library_dir, path) @@ -107,8 +111,7 @@ class VCR(object): 'ignore_localhost', self.ignore_localhost ), } - - return Cassette.use(path, **merged_config) + return path, merged_config def register_serializer(self, name, serializer): self.serializers[name] = serializer From 9a564586a4a05b1c11a4166720d1d4599012b1c7 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 03:46:39 -0700 Subject: [PATCH 10/29] Failing tests for nested context decoration. --- tests/unit/test_cassettes.py | 51 +++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index c0f415b..7b665dc 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -1,3 +1,4 @@ +import copy from six.moves import http_client as httplib import pytest @@ -71,17 +72,18 @@ def test_cassette_cant_read_same_request_twice(): a.play_response('foo') +def make_get_request(): + conn = httplib.HTTPConnection("www.python.org") + conn.request("GET", "/index.html") + return conn.getresponse() + + @mock.patch('vcr.cassette.requests_match', return_value=True) @mock.patch('vcr.cassette.load_cassette', lambda *args, **kwargs: (('foo',), (mock.MagicMock(),))) @mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) @mock.patch('vcr.stubs.VCRHTTPResponse') def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): - @Cassette.use('test') - def decorated_function(): - conn = httplib.HTTPConnection("www.python.org") - conn.request("GET", "/index.html") - conn.getresponse() - + decorated_function = Cassette.use('test')(make_get_request) for i in range(2): decorated_function() @@ -133,3 +135,40 @@ def test_before_record_response(): before_record_response.assert_called_once_with('res') assert cassette.responses[0] == 'mutated' + + +def assert_get_response_body_is(value): + conn = httplib.HTTPConnection("www.python.org") + conn.request("GET", "/index.html") + assert conn.getresponse().read().decode('utf8') == value + + +@mock.patch('vcr.cassette.requests_match', _mock_requests_match) +@mock.patch('vcr.cassette.Cassette.can_play_response_for', return_value=True) +@mock.patch('vcr.cassette.Cassette._save', return_value=True) +def test_nesting_cassette_context_managers(*args): + first_response = {'body': {'string': b'first_response'}, 'headers': {}, + 'status': {'message': 'm', 'code': 200}} + + second_response = copy.deepcopy(first_response) + second_response['body']['string'] = b'second_response' + + with Cassette.use('test') as first_cassette, \ + mock.patch.object(first_cassette, 'play_response', return_value=first_response): + assert_get_response_body_is('first_response') + + # Make sure a second cassette can supercede the first + with Cassette.use('test') as second_cassette, \ + mock.patch.object(second_cassette, 'play_response', return_value=second_response): + assert_get_response_body_is('second_response') + + # Now the first cassette should be back in effect + assert_get_response_body_is('first_response') + + +def test_nesting_context_managers_by_checking_references_of_http_connection(): + with Cassette.use('test'): + first_cassette_HTTPConnection = httplib.HTTPConnection + with Cassette.use('test'): + pass + assert httplib.HTTPConnection is first_cassette_HTTPConnection From 958aac3af3ea1a85594def6c62fedde2a5315d12 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 05:32:55 -0700 Subject: [PATCH 11/29] Use mock for patching http connection objects. --- setup.py | 2 +- tests/unit/test_cassettes.py | 33 +++++--- tests/unit/test_vcr.py | 10 +-- vcr/cassette.py | 46 +++++------ vcr/patch.py | 152 ++++++++++++++++++++--------------- vcr/stubs/__init__.py | 7 +- 6 files changed, 138 insertions(+), 112 deletions(-) diff --git a/setup.py b/setup.py index 0540e85..3e1fc05 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( 'vcr.compat': 'vcr/compat', 'vcr.persisters': 'vcr/persisters', }, - install_requires=['PyYAML', 'six'], + install_requires=['PyYAML', 'mock', 'six', 'contextlib2'], license='MIT', tests_require=['pytest', 'mock', 'pytest-localserver'], cmdclass={'test': PyTest}, diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 7b665dc..2cab15f 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -1,11 +1,13 @@ import copy -from six.moves import http_client as httplib +from six.moves import http_client as httplib +import contextlib2 +import mock import pytest import yaml -import mock from vcr.cassette import Cassette +from vcr.patch import force_reset from vcr.errors import UnhandledHTTPRequestError @@ -106,11 +108,9 @@ def test_arg_getter_functionality(): def function(): pass - with mock.patch.object(Cassette, '__init__', return_value=None) as cassette_init, \ - mock.patch.object(Cassette, '_load'), \ - mock.patch.object(Cassette, '__exit__'): + with mock.patch.object(Cassette, 'load') as cassette_load: function() - cassette_init.assert_called_once_with(arg_getter.return_value[0], + cassette_load.assert_called_once_with(arg_getter.return_value[0], **arg_getter.return_value[1]) @@ -153,22 +153,31 @@ def test_nesting_cassette_context_managers(*args): second_response = copy.deepcopy(first_response) second_response['body']['string'] = b'second_response' - with Cassette.use('test') as first_cassette, \ - mock.patch.object(first_cassette, 'play_response', return_value=first_response): + with contextlib2.ExitStack() as exit_stack: + first_cassette = exit_stack.enter_context(Cassette.use('test')) + exit_stack.enter_context(mock.patch.object(first_cassette, 'play_response', + return_value=first_response)) assert_get_response_body_is('first_response') # Make sure a second cassette can supercede the first - with Cassette.use('test') as second_cassette, \ - mock.patch.object(second_cassette, 'play_response', return_value=second_response): - assert_get_response_body_is('second_response') + with Cassette.use('test') as second_cassette: + with mock.patch.object(second_cassette, 'play_response', return_value=second_response): + assert_get_response_body_is('second_response') # Now the first cassette should be back in effect assert_get_response_body_is('first_response') def test_nesting_context_managers_by_checking_references_of_http_connection(): + original = httplib.HTTPConnection with Cassette.use('test'): first_cassette_HTTPConnection = httplib.HTTPConnection with Cassette.use('test'): - pass + second_cassette_HTTPConnection = httplib.HTTPConnection + assert second_cassette_HTTPConnection is not first_cassette_HTTPConnection + with Cassette.use('test'): + assert httplib.HTTPConnection is not second_cassette_HTTPConnection + with force_reset(): + assert httplib.HTTPConnection is original + assert httplib.HTTPConnection is second_cassette_HTTPConnection assert httplib.HTTPConnection is first_cassette_HTTPConnection diff --git a/tests/unit/test_vcr.py b/tests/unit/test_vcr.py index d460bc2..c1dc6f0 100644 --- a/tests/unit/test_vcr.py +++ b/tests/unit/test_vcr.py @@ -6,21 +6,19 @@ from vcr import VCR def test_vcr_use_cassette(): filter_headers = mock.Mock() test_vcr = VCR(filter_headers=filter_headers) - with mock.patch('vcr.cassette.Cassette.__init__', return_value=None) as mock_cassette_init, \ - mock.patch('vcr.cassette.Cassette._load'), \ - mock.patch('vcr.cassette.Cassette.__exit__'): + with mock.patch('vcr.cassette.Cassette.load') as mock_cassette_load: @test_vcr.use_cassette('test') def function(): pass - assert mock_cassette_init.call_count == 0 + assert mock_cassette_load.call_count == 0 function() - assert mock_cassette_init.call_args[1]['filter_headers'] is filter_headers + assert mock_cassette_load.call_args[1]['filter_headers'] is filter_headers # Make sure that calls to function now use cassettes with the # new filter_header_settings test_vcr.filter_headers = ('a',) function() - assert mock_cassette_init.call_args[1]['filter_headers'] == test_vcr.filter_headers + assert mock_cassette_load.call_args[1]['filter_headers'] == test_vcr.filter_headers # Ensure that explicitly provided arguments still supercede # those on the vcr. diff --git a/vcr/cassette.py b/vcr/cassette.py index cb46e35..8949c19 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -1,4 +1,5 @@ '''The container for recorded requests and responses''' +import contextlib2 import functools try: from collections import Counter @@ -6,7 +7,7 @@ except ImportError: from .compat.counter import Counter # Internal imports -from .patch import install, reset +from .patch import build_patchers from .persist import load_cassette, save_cassette from .filters import filter_request from .serializers import yamlserializer @@ -31,14 +32,27 @@ class CassetteContextDecorator(object): def __init__(self, cls, args_getter): self.cls = cls self._args_getter = args_getter + self.__finish = None + + def _patch_generator(self, cassette): + with contextlib2.ExitStack() as exit_stack: + for patcher in build_patchers(cassette): + exit_stack.enter_context(patcher) + yield + # TODO(@IvanMalison): Hmmm. it kind of feels like this should be somewhere else. + cassette._save() def __enter__(self): + assert self.__finish is None path, kwargs = self._args_getter() - self._cassette = self.cls.load(path, **kwargs) - return self._cassette.__enter__() + cassette = self.cls.load(path, **kwargs) + self.__finish = self._patch_generator(cassette) + self.__finish.__next__() + return cassette def __exit__(self, *args): - return self._cassette.__exit__(*args) + [_ for _ in self.__finish] # this exits the context created by the call to _patch_generator. + self.__finish = None def __call__(self, function): @functools.wraps(function) @@ -66,17 +80,10 @@ class Cassette(object): def use(cls, *args, **kwargs): return CassetteContextDecorator.from_args(cls, *args, **kwargs) - def __init__(self, - path, - serializer=yamlserializer, - record_mode='once', - match_on=(uri, method), - filter_headers=(), - filter_query_parameters=(), - before_record=None, - before_record_response=None, - ignore_hosts=(), - ignore_localhost=()): + def __init__(self, path, serializer=yamlserializer, record_mode='once', + match_on=(uri, method), filter_headers=(), + filter_query_parameters=(), before_record=None, before_record_response=None, + ignore_hosts=(), ignore_localhost=()): self._path = path self._serializer = serializer self._match_on = match_on @@ -228,12 +235,3 @@ class Cassette(object): for response in self._responses(request): return True return False - - def __enter__(self): - '''Patch the fetching libraries we know about''' - install(self) - return self - - def __exit__(self, typ, value, traceback): - self._save() - reset() diff --git a/vcr/patch.py b/vcr/patch.py index 2d40362..4c76269 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -1,4 +1,6 @@ '''Utilities for patching in cassettes''' +import contextlib2 +import mock from .stubs import VCRHTTPConnection, VCRHTTPSConnection from six.moves import http_client as httplib @@ -8,139 +10,159 @@ from six.moves import http_client as httplib _HTTPConnection = httplib.HTTPConnection _HTTPSConnection = httplib.HTTPSConnection + +# Try to save the original types for requests try: - # Try to save the original types for requests import requests.packages.urllib3.connectionpool as cpool +except ImportError: # pragma: no cover + pass +else: _VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection _cpoolHTTPConnection = cpool.HTTPConnection _cpoolHTTPSConnection = cpool.HTTPSConnection -except ImportError: # pragma: no cover - pass + +# Try to save the original types for urllib3 try: - # Try to save the original types for urllib3 import urllib3 - _VerifiedHTTPSConnection = urllib3.connectionpool.VerifiedHTTPSConnection except ImportError: # pragma: no cover pass +else: + _VerifiedHTTPSConnection = urllib3.connectionpool.VerifiedHTTPSConnection + +# Try to save the original types for httplib2 try: - # Try to save the original types for httplib2 import httplib2 +except ImportError: # pragma: no cover + pass +else: _HTTPConnectionWithTimeout = httplib2.HTTPConnectionWithTimeout _HTTPSConnectionWithTimeout = httplib2.HTTPSConnectionWithTimeout _SCHEME_TO_CONNECTION = httplib2.SCHEME_TO_CONNECTION -except ImportError: # pragma: no cover - pass + +# Try to save the original types for boto try: - # Try to save the original types for boto import boto.https_connection - _CertValidatingHTTPSConnection = \ - boto.https_connection.CertValidatingHTTPSConnection except ImportError: # pragma: no cover pass +else: + _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection -def install(cassette): + +def cassette_subclass(base_class, cassette): + return type('{0}{1}'.format(base_class.__name__, cassette._path), + (base_class,), dict(cassette=cassette)) + + +def build_patchers(cassette): """ - Patch all the HTTPConnections references we can find! + Build patches for all the HTTPConnections references we can find! This replaces the actual HTTPConnection with a VCRHTTPConnection object which knows how to save to / read from cassettes """ - httplib.HTTPConnection = VCRHTTPConnection - httplib.HTTPSConnection = VCRHTTPSConnection - httplib.HTTPConnection.cassette = cassette - httplib.HTTPSConnection.cassette = cassette + _VCRHTTPConnection = cassette_subclass(VCRHTTPConnection, cassette) + _VCRHTTPSConnection = cassette_subclass(VCRHTTPSConnection, cassette) - # patch requests v1.x + + yield mock.patch.object(httplib, 'HTTPConnection', _VCRHTTPConnection) + yield mock.patch.object(httplib, 'HTTPSConnection', _VCRHTTPSConnection) + + # requests try: import requests.packages.urllib3.connectionpool as cpool - from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection - cpool.VerifiedHTTPSConnection = VCRRequestsHTTPSConnection - cpool.HTTPConnection = VCRRequestsHTTPConnection - cpool.VerifiedHTTPSConnection.cassette = cassette - cpool.HTTPConnection = VCRHTTPConnection - cpool.HTTPConnection.cassette = cassette - # patch requests v2.x - cpool.HTTPConnectionPool.ConnectionCls = VCRRequestsHTTPConnection - cpool.HTTPConnectionPool.cassette = cassette - cpool.HTTPSConnectionPool.ConnectionCls = VCRRequestsHTTPSConnection - cpool.HTTPSConnectionPool.cassette = cassette except ImportError: # pragma: no cover pass + else: + from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + # patch requests v1.x + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', cassette_subclass(VCRRequestsHTTPSConnection, cassette)) + yield mock.patch.object(cpool, 'HTTPConnection', cassette_subclass(VCRRequestsHTTPConnection, cassette)) + yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) + # patch requests v2.x + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', cassette_subclass(VCRRequestsHTTPConnection, cassette)) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', cassette_subclass(VCRRequestsHTTPSConnection, cassette)) # patch urllib3 try: import urllib3.connectionpool as cpool - from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection - cpool.VerifiedHTTPSConnection = VCRVerifiedHTTPSConnection - cpool.VerifiedHTTPSConnection.cassette = cassette - cpool.HTTPConnection = VCRHTTPConnection - cpool.HTTPConnection.cassette = cassette except ImportError: # pragma: no cover pass + else: + from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', cassette_subclass(VCRVerifiedHTTPSConnection, cassette)) + yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) # patch httplib2 try: import httplib2 as cpool - from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout - from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout - cpool.HTTPConnectionWithTimeout = VCRHTTPConnectionWithTimeout - cpool.HTTPSConnectionWithTimeout = VCRHTTPSConnectionWithTimeout - cpool.SCHEME_TO_CONNECTION = { - 'http': VCRHTTPConnectionWithTimeout, - 'https': VCRHTTPSConnectionWithTimeout - } except ImportError: # pragma: no cover pass + else: + from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout + from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout + yield mock.patch.object(cpool, 'HTTPConnectionWithTimeout', cassette_subclass(VCRHTTPConnectionWithTimeout, cassette)) + yield mock.patch.object(cpool, 'HTTPSConnectionWithTimeout', cassette_subclass(VCRHTTPSConnectionWithTimeout, cassette)) + yield mock.patch.object(cpool, 'SCHEME_TO_CONNECTION', {'http': VCRHTTPConnectionWithTimeout, 'https': VCRHTTPSConnectionWithTimeout}) # patch boto try: import boto.https_connection as cpool - from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection - cpool.CertValidatingHTTPSConnection = VCRCertValidatingHTTPSConnection - cpool.CertValidatingHTTPSConnection.cassette = cassette except ImportError: # pragma: no cover pass + else: + from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection + yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', cassette_subclass(VCRCertValidatingHTTPSConnection, cassette)) -def reset(): - '''Undo all the patching''' - httplib.HTTPConnection = _HTTPConnection - httplib.HTTPSConnection = _HTTPSConnection +def reset_patchers(): + yield mock.patch.object(httplib, 'HTTPConnection', _HTTPConnection) + yield mock.patch.object(httplib, 'HTTPSConnection', _HTTPSConnection) try: import requests.packages.urllib3.connectionpool as cpool - # unpatch requests v1.x - cpool.VerifiedHTTPSConnection = _VerifiedHTTPSConnection - cpool.HTTPConnection = _cpoolHTTPConnection - # unpatch requests v2.x - cpool.HTTPConnectionPool.ConnectionCls = _cpoolHTTPConnection - cpool.HTTPSConnection = _cpoolHTTPSConnection - cpool.HTTPSConnectionPool.ConnectionCls = _cpoolHTTPSConnection except ImportError: # pragma: no cover pass + else: + # unpatch requests v1.x + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _VerifiedHTTPSConnection) + yield mock.patch.object(cpool, 'HTTPConnection', _cpoolHTTPConnection) + # unpatch requests v2.x + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', _cpoolHTTPConnection) + yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolHTTPSConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _cpoolHTTPSConnection) try: import urllib3.connectionpool as cpool - cpool.VerifiedHTTPSConnection = _VerifiedHTTPSConnection - cpool.HTTPConnection = _HTTPConnection - cpool.HTTPSConnection = _HTTPSConnection - cpool.HTTPConnectionPool.ConnectionCls = _HTTPConnection - cpool.HTTPSConnectionPool.ConnectionCls = _HTTPSConnection except ImportError: # pragma: no cover pass + else: + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _VerifiedHTTPSConnection) + yield mock.patch.object(cpool, 'HTTPConnection', _HTTPConnection) + yield mock.patch.object(cpool, 'HTTPSConnection', _HTTPSConnection) + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', _HTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _HTTPSConnection) try: import httplib2 as cpool - cpool.HTTPConnectionWithTimeout = _HTTPConnectionWithTimeout - cpool.HTTPSConnectionWithTimeout = _HTTPSConnectionWithTimeout - cpool.SCHEME_TO_CONNECTION = _SCHEME_TO_CONNECTION except ImportError: # pragma: no cover pass + else: + yield mock.patch.object(cpool, 'HTTPConnectionWithTimeout', _HTTPConnectionWithTimeout) + yield mock.patch.object(cpool, 'HTTPSConnectionWithTimeout', _HTTPSConnectionWithTimeout) + yield mock.patch.object(cpool, 'SCHEME_TO_CONNECTION', _SCHEME_TO_CONNECTION) try: import boto.https_connection as cpool - cpool.CertValidatingHTTPSConnection = _CertValidatingHTTPSConnection except ImportError: # pragma: no cover pass + else: + yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', _CertValidatingHTTPSConnection) + +@contextlib2.contextmanager +def force_reset(): + with contextlib2.ExitStack() as exit_stack: + for patcher in reset_patchers(): + exit_stack.enter_context(patcher) + yield diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 45af7ea..772731e 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -295,10 +295,9 @@ class VCRConnection: # need to temporarily reset here because the real connection # inherits from the thing that we are mocking out. Take out # the reset if you want to see what I mean :) - from vcr.patch import install, reset - reset() - self.real_connection = self._baseclass(*args, **kwargs) - install(self.cassette) + from vcr.patch import force_reset + with force_reset(): + self.real_connection = self._baseclass(*args, **kwargs) class VCRHTTPConnection(VCRConnection): From 21930081506972a7a4cee7fee0614fe44362dc8c Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 05:59:03 -0700 Subject: [PATCH 12/29] Python version agnostic way of getting the next item in the generator. --- vcr/cassette.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index 8949c19..e5e58ad 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -38,20 +38,18 @@ class CassetteContextDecorator(object): with contextlib2.ExitStack() as exit_stack: for patcher in build_patchers(cassette): exit_stack.enter_context(patcher) - yield + yield cassette # TODO(@IvanMalison): Hmmm. it kind of feels like this should be somewhere else. cassette._save() def __enter__(self): assert self.__finish is None path, kwargs = self._args_getter() - cassette = self.cls.load(path, **kwargs) - self.__finish = self._patch_generator(cassette) - self.__finish.__next__() - return cassette + self.__finish = self._patch_generator(self.cls.load(path, **kwargs)) + return next(self.__finish) def __exit__(self, *args): - [_ for _ in self.__finish] # this exits the context created by the call to _patch_generator. + next(self.__finish, None) self.__finish = None def __call__(self, function): From 5edc58f10c60e4437573da568d676677be06be10 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 06:48:55 -0700 Subject: [PATCH 13/29] Check for old style class when building subclass. --- vcr/patch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index 4c76269..7da13d0 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -1,4 +1,6 @@ '''Utilities for patching in cassettes''' +import types + import contextlib2 import mock @@ -53,8 +55,10 @@ else: def cassette_subclass(base_class, cassette): - return type('{0}{1}'.format(base_class.__name__, cassette._path), - (base_class,), dict(cassette=cassette)) + bases = (base_class,) + if not issubclass(base_class, object): # Check for old style class + bases += (object,) + return type('{0}{1}'.format(base_class.__name__, cassette._path), bases, dict(cassette=cassette)) def build_patchers(cassette): From e1e08c7a2cbeeeb4ee9c29841e0bf42ecc0f232b Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 08:02:50 -0700 Subject: [PATCH 14/29] hasattr check for requests 2.0 use cassette added type for httplib2 dictionary patch. --- vcr/patch.py | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index 7da13d0..35f9b23 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -1,6 +1,4 @@ '''Utilities for patching in cassettes''' -import types - import contextlib2 import mock @@ -81,13 +79,20 @@ def build_patchers(cassette): pass else: from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + # patch requests v1.x - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', cassette_subclass(VCRRequestsHTTPSConnection, cassette)) - yield mock.patch.object(cpool, 'HTTPConnection', cassette_subclass(VCRRequestsHTTPConnection, cassette)) + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', + cassette_subclass(VCRRequestsHTTPSConnection, cassette)) + yield mock.patch.object(cpool, 'HTTPConnection', + cassette_subclass(VCRRequestsHTTPConnection, cassette)) yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) + # patch requests v2.x - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', cassette_subclass(VCRRequestsHTTPConnection, cassette)) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', cassette_subclass(VCRRequestsHTTPSConnection, cassette)) + if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + cassette_subclass(VCRRequestsHTTPConnection, cassette)) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + cassette_subclass(VCRRequestsHTTPSConnection, cassette)) # patch urllib3 try: @@ -96,7 +101,8 @@ def build_patchers(cassette): pass else: from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', cassette_subclass(VCRVerifiedHTTPSConnection, cassette)) + yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', + cassette_subclass(VCRVerifiedHTTPSConnection, cassette)) yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) # patch httplib2 @@ -107,9 +113,17 @@ def build_patchers(cassette): else: from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout - yield mock.patch.object(cpool, 'HTTPConnectionWithTimeout', cassette_subclass(VCRHTTPConnectionWithTimeout, cassette)) - yield mock.patch.object(cpool, 'HTTPSConnectionWithTimeout', cassette_subclass(VCRHTTPSConnectionWithTimeout, cassette)) - yield mock.patch.object(cpool, 'SCHEME_TO_CONNECTION', {'http': VCRHTTPConnectionWithTimeout, 'https': VCRHTTPSConnectionWithTimeout}) + _VCRHTTPConnectionWithTimeout = cassette_subclass(VCRHTTPConnectionWithTimeout, + cassette) + _VCRHTTPSConnectionWithTimeout = cassette_subclass(VCRHTTPSConnectionWithTimeout, + cassette) + yield mock.patch.object(cpool, 'HTTPConnectionWithTimeout', + _VCRHTTPConnectionWithTimeout) + yield mock.patch.object(cpool, 'HTTPSConnectionWithTimeout', + _VCRHTTPSConnectionWithTimeout) + yield mock.patch.object(cpool, 'SCHEME_TO_CONNECTION', + {'http': _VCRHTTPConnectionWithTimeout, + 'https': _VCRHTTPSConnectionWithTimeout}) # patch boto try: @@ -118,7 +132,8 @@ def build_patchers(cassette): pass else: from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection - yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', cassette_subclass(VCRCertValidatingHTTPSConnection, cassette)) + yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', + cassette_subclass(VCRCertValidatingHTTPSConnection, cassette)) def reset_patchers(): @@ -133,9 +148,11 @@ def reset_patchers(): yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _VerifiedHTTPSConnection) yield mock.patch.object(cpool, 'HTTPConnection', _cpoolHTTPConnection) # unpatch requests v2.x - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', _cpoolHTTPConnection) + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + _cpoolHTTPConnection) yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolHTTPSConnection) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', _cpoolHTTPSConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + _cpoolHTTPSConnection) try: import urllib3.connectionpool as cpool @@ -162,7 +179,9 @@ def reset_patchers(): except ImportError: # pragma: no cover pass else: - yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', _CertValidatingHTTPSConnection) + yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', + _CertValidatingHTTPSConnection) + @contextlib2.contextmanager def force_reset(): From 4868a63876ad34e71e77e224d05100584dc205b8 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 09:24:28 -0700 Subject: [PATCH 15/29] Refactor build_patchers into class. Fix issue with patching non existent attribute with hasattr. --- vcr/cassette.py | 4 +- vcr/patch.py | 166 ++++++++++++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 79 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index e5e58ad..f03c01b 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -7,7 +7,7 @@ except ImportError: from .compat.counter import Counter # Internal imports -from .patch import build_patchers +from .patch import PatcherBuilder from .persist import load_cassette, save_cassette from .filters import filter_request from .serializers import yamlserializer @@ -36,7 +36,7 @@ class CassetteContextDecorator(object): def _patch_generator(self, cassette): with contextlib2.ExitStack() as exit_stack: - for patcher in build_patchers(cassette): + for patcher in PatcherBuilder(cassette).build_patchers(): exit_stack.enter_context(patcher) yield cassette # TODO(@IvanMalison): Hmmm. it kind of feels like this should be somewhere else. diff --git a/vcr/patch.py b/vcr/patch.py index 35f9b23..e38cea0 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -1,4 +1,6 @@ '''Utilities for patching in cassettes''' +import itertools + import contextlib2 import mock @@ -51,89 +53,96 @@ else: _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection +class PatcherBuilder(object): -def cassette_subclass(base_class, cassette): - bases = (base_class,) - if not issubclass(base_class, object): # Check for old style class - bases += (object,) - return type('{0}{1}'.format(base_class.__name__, cassette._path), bases, dict(cassette=cassette)) + def __init__(self, cassette): + self._cassette = cassette + self._class_to_cassette_subclass = {} + def build_patchers(self): + patcher_args = itertools.chain(self._httplib(), self._requests(), self._urllib3(), + self._httplib2(), self._boto()) + for args in patcher_args: + patcher = self._build_patcher(*args) + if patcher: + yield patcher -def build_patchers(cassette): - """ - Build patches for all the HTTPConnections references we can find! - This replaces the actual HTTPConnection with a VCRHTTPConnection - object which knows how to save to / read from cassettes - """ - _VCRHTTPConnection = cassette_subclass(VCRHTTPConnection, cassette) - _VCRHTTPSConnection = cassette_subclass(VCRHTTPSConnection, cassette) + def _build_patcher(self, obj, patched_attribute, replacement_class): + if not hasattr(obj, patched_attribute): + return + if isinstance(replacement_class, dict): + for key in replacement_class: + replacement_class[key] = self._get_cassette_subclass(replacement_class[key]) + else: + replacement_class = self._get_cassette_subclass(replacement_class) + return mock.patch.object(obj, patched_attribute, replacement_class) - yield mock.patch.object(httplib, 'HTTPConnection', _VCRHTTPConnection) - yield mock.patch.object(httplib, 'HTTPSConnection', _VCRHTTPSConnection) + def _get_cassette_subclass(self, klass): + if klass not in self._class_to_cassette_subclass: + self._class_to_cassette_subclass[klass] = self._cassette_subclass(klass) + return self._class_to_cassette_subclass[klass] - # requests - try: - import requests.packages.urllib3.connectionpool as cpool - except ImportError: # pragma: no cover - pass - else: - from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + def _cassette_subclass(self, base_class): + bases = (base_class,) + if not issubclass(base_class, object): # Check for old style class + bases += (object,) + return type('{0}{1}'.format(base_class.__name__, self._cassette._path), + bases, dict(cassette=self._cassette)) - # patch requests v1.x - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', - cassette_subclass(VCRRequestsHTTPSConnection, cassette)) - yield mock.patch.object(cpool, 'HTTPConnection', - cassette_subclass(VCRRequestsHTTPConnection, cassette)) - yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) + def _httplib(self): + yield httplib, 'HTTPConnection', VCRHTTPConnection + yield httplib, 'HTTPSConnection', VCRHTTPSConnection - # patch requests v2.x - if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', - cassette_subclass(VCRRequestsHTTPConnection, cassette)) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', - cassette_subclass(VCRRequestsHTTPSConnection, cassette)) + def _requests(self): + try: + import requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection - # patch urllib3 - try: - import urllib3.connectionpool as cpool - except ImportError: # pragma: no cover - pass - else: - from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection - yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', - cassette_subclass(VCRVerifiedHTTPSConnection, cassette)) - yield mock.patch.object(cpool, 'HTTPConnection', _VCRHTTPConnection) + yield cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection + yield cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection + yield cpool, 'HTTPConnection', VCRRequestsHTTPConnection + yield cpool, 'HTTPConnection', VCRHTTPConnection + yield cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection + yield cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection - # patch httplib2 - try: - import httplib2 as cpool - except ImportError: # pragma: no cover - pass - else: - from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout - from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout - _VCRHTTPConnectionWithTimeout = cassette_subclass(VCRHTTPConnectionWithTimeout, - cassette) - _VCRHTTPSConnectionWithTimeout = cassette_subclass(VCRHTTPSConnectionWithTimeout, - cassette) - yield mock.patch.object(cpool, 'HTTPConnectionWithTimeout', - _VCRHTTPConnectionWithTimeout) - yield mock.patch.object(cpool, 'HTTPSConnectionWithTimeout', - _VCRHTTPSConnectionWithTimeout) - yield mock.patch.object(cpool, 'SCHEME_TO_CONNECTION', - {'http': _VCRHTTPConnectionWithTimeout, - 'https': _VCRHTTPSConnectionWithTimeout}) + def _urllib3(self): + try: + import urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection + + yield cpool, 'VerifiedHTTPSConnection', VCRVerifiedHTTPSConnection + yield cpool, 'HTTPConnection', VCRHTTPConnection + + def _httplib2(self): + try: + import httplib2 as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout + from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout + + yield cpool, 'HTTPConnectionWithTimeout', VCRHTTPConnectionWithTimeout + yield cpool, 'HTTPSConnectionWithTimeout', VCRHTTPSConnectionWithTimeout + yield cpool, 'SCHEME_TO_CONNECTION', {'http': VCRHTTPConnectionWithTimeout, + 'https': VCRHTTPSConnectionWithTimeout} + + def _boto(self): + try: + import boto.https_connection as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection + yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection - # patch boto - try: - import boto.https_connection as cpool - except ImportError: # pragma: no cover - pass - else: - from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection - yield mock.patch.object(cpool, 'CertValidatingHTTPSConnection', - cassette_subclass(VCRCertValidatingHTTPSConnection, cassette)) def reset_patchers(): @@ -148,11 +157,14 @@ def reset_patchers(): yield mock.patch.object(cpool, 'VerifiedHTTPSConnection', _VerifiedHTTPSConnection) yield mock.patch.object(cpool, 'HTTPConnection', _cpoolHTTPConnection) # unpatch requests v2.x - yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', - _cpoolHTTPConnection) - yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolHTTPSConnection) - yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', - _cpoolHTTPSConnection) + if hasattr(cpool.HTTPConnectionPool, 'ConnectionCls'): + yield mock.patch.object(cpool.HTTPConnectionPool, 'ConnectionCls', + _cpoolHTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, 'ConnectionCls', + _cpoolHTTPSConnection) + + if hasattr(cpool, 'HTTPSConnection'): + yield mock.patch.object(cpool, 'HTTPSConnection', _cpoolHTTPSConnection) try: import urllib3.connectionpool as cpool From 0c19acd74f2cfc8c4b351021a490bc32a33af8e1 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 15:25:42 -0700 Subject: [PATCH 16/29] Use contextdecorator from contextlib2. add logging for entering context. --- vcr/cassette.py | 17 ++++++++--------- vcr/patch.py | 7 ++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index f03c01b..ddbb0f1 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -1,6 +1,7 @@ '''The container for recorded requests and responses''' +import logging + import contextlib2 -import functools try: from collections import Counter except ImportError: @@ -15,7 +16,10 @@ from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError -class CassetteContextDecorator(object): +log = logging.getLogger(__name__) + + +class CassetteContextDecorator(contextlib2.ContextDecorator): """Context manager/decorator that handles installing the cassette and removing cassettes. @@ -38,7 +42,9 @@ class CassetteContextDecorator(object): with contextlib2.ExitStack() as exit_stack: for patcher in PatcherBuilder(cassette).build_patchers(): exit_stack.enter_context(patcher) + log.debug('Entered context for cassette at {0}.'.format(cassette._path)) yield cassette + log.debug('Exiting context for cassette at {0}.'.format(cassette._path)) # TODO(@IvanMalison): Hmmm. it kind of feels like this should be somewhere else. cassette._save() @@ -52,13 +58,6 @@ class CassetteContextDecorator(object): next(self.__finish, None) self.__finish = None - def __call__(self, function): - @functools.wraps(function) - def wrapped(*args, **kwargs): - with self: - return function(*args, **kwargs) - return wrapped - class Cassette(object): '''A container for recorded requests and responses''' diff --git a/vcr/patch.py b/vcr/patch.py index e38cea0..dc4e72e 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -79,11 +79,13 @@ class PatcherBuilder(object): return mock.patch.object(obj, patched_attribute, replacement_class) def _get_cassette_subclass(self, klass): + if klass.cassette is not None: + return klass if klass not in self._class_to_cassette_subclass: - self._class_to_cassette_subclass[klass] = self._cassette_subclass(klass) + self._class_to_cassette_subclass[klass] = self._build_cassette_subclass(klass) return self._class_to_cassette_subclass[klass] - def _cassette_subclass(self, base_class): + def _build_cassette_subclass(self, base_class): bases = (base_class,) if not issubclass(base_class, object): # Check for old style class bases += (object,) @@ -144,7 +146,6 @@ class PatcherBuilder(object): yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection - def reset_patchers(): yield mock.patch.object(httplib, 'HTTPConnection', _HTTPConnection) yield mock.patch.object(httplib, 'HTTPSConnection', _HTTPSConnection) From 58fcb2b453121e2593117cead26e0bd79e08f0b0 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 16:17:48 -0700 Subject: [PATCH 17/29] Add test that fails because of the fact that a new class is used for each cassette context instead of replacing the cassette on the existing mock connection. --- tests/integration/test_requests.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index 3613155..3139ac5 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -146,6 +146,31 @@ def test_session_and_connection_close(tmpdir, scheme): resp = session.get('http://httpbin.org/get', headers={'Connection': 'close'}) resp = session.get('http://httpbin.org/get', headers={'Connection': 'close'}) + def test_https_with_cert_validation_disabled(tmpdir): with vcr.use_cassette(str(tmpdir.join('cert_validation_disabled.yaml'))): requests.get('https://httpbin.org', verify=False) + + +def test_nested_context_managers_with_session_created_before_first_nesting(scheme, tmpdir): + ''' + This tests ensures that a session that was created while one cassette was + ''' + url = scheme + '://httpbin.org/bytes/1024' + with vcr.use_cassette(str(tmpdir.join('first_nested.yaml'))): + session = requests.session() + first_body = session.get(url).content + with vcr.use_cassette(str(tmpdir.join('second_nested.yaml'))): + second_body = session.get(url).content + third_body = requests.get(url).content + + with vcr.use_cassette(str(tmpdir.join('second_nested.yaml'))): + session = requests.session() + assert session.get(url).content == second_body + with vcr.use_cassette(str(tmpdir.join('first_nested.yaml'))): + assert session.get(url).content == first_body + assert session.get(url).content == third_body + + # Make sure that the session can now get content normally. + session.get('http://www.reddit.com') + From 2bf23b2cdf8542f5bf137e386f3bc278bbc2cc83 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 17:01:48 -0700 Subject: [PATCH 18/29] Fixed issue in test_nested_context_managers_with_session_created_before_first_nesting. by using a single class and patching cassette on that class. Not a great solution :\ --- vcr/patch.py | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index dc4e72e..ab0036b 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -57,41 +57,22 @@ class PatcherBuilder(object): def __init__(self, cassette): self._cassette = cassette - self._class_to_cassette_subclass = {} def build_patchers(self): patcher_args = itertools.chain(self._httplib(), self._requests(), self._urllib3(), self._httplib2(), self._boto()) - for args in patcher_args: - patcher = self._build_patcher(*args) + for obj, patched_attribute, replacement_class in patcher_args: + patcher = self._build_patcher(obj, patched_attribute, replacement_class) if patcher: yield patcher + if hasattr(replacement_class, 'cassette'): + yield mock.patch.object(replacement_class, 'cassette', self._cassette) def _build_patcher(self, obj, patched_attribute, replacement_class): if not hasattr(obj, patched_attribute): return - - if isinstance(replacement_class, dict): - for key in replacement_class: - replacement_class[key] = self._get_cassette_subclass(replacement_class[key]) - else: - replacement_class = self._get_cassette_subclass(replacement_class) return mock.patch.object(obj, patched_attribute, replacement_class) - def _get_cassette_subclass(self, klass): - if klass.cassette is not None: - return klass - if klass not in self._class_to_cassette_subclass: - self._class_to_cassette_subclass[klass] = self._build_cassette_subclass(klass) - return self._class_to_cassette_subclass[klass] - - def _build_cassette_subclass(self, base_class): - bases = (base_class,) - if not issubclass(base_class, object): # Check for old style class - bases += (object,) - return type('{0}{1}'.format(base_class.__name__, self._cassette._path), - bases, dict(cassette=self._cassette)) - def _httplib(self): yield httplib, 'HTTPConnection', VCRHTTPConnection yield httplib, 'HTTPSConnection', VCRHTTPSConnection From 8947f0fc5c1ec2d1a238351f419fe247b4f002c5 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Thu, 18 Sep 2014 17:03:13 -0700 Subject: [PATCH 19/29] Add failing test for session still being attached to cassette after context is gone. --- tests/integration/test_requests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index 3139ac5..e36da18 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -152,6 +152,18 @@ def test_https_with_cert_validation_disabled(tmpdir): requests.get('https://httpbin.org', verify=False) +def test_session_can_make_requests_after_requests_unpatched(tmpdir): + with vcr.use_cassette(str(tmpdir.join('test_session_after_unpatched.yaml'))): + session = requests.session() + session.get('http://httpbin.org/get') + + with vcr.use_cassette(str(tmpdir.join('test_session_after_unpatched.yaml'))): + session = requests.session() + session.get('http://httpbin.org/get') + + session.get('http://httpbin.org/status/200') + + def test_nested_context_managers_with_session_created_before_first_nesting(scheme, tmpdir): ''' This tests ensures that a session that was created while one cassette was From b6e96020c159d5d8f6666de064dc3c2608d6637b Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 19 Sep 2014 14:31:49 -0700 Subject: [PATCH 20/29] Use {[testenv]deps}, instead of repeating testing requirements. Write another failing test for #109 --- tests/integration/test_requests.py | 53 +++++++++---- tox.ini | 122 ++++++----------------------- vcr/stubs/__init__.py | 4 +- 3 files changed, 64 insertions(+), 115 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index e36da18..9062d7d 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -24,30 +24,30 @@ def scheme(request): def test_status_code(scheme, tmpdir): '''Ensure that we can read the status code''' url = scheme + '://httpbin.org/' - with vcr.use_cassette(str(tmpdir.join('atts.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('atts.yaml'))): status_code = requests.get(url).status_code - with vcr.use_cassette(str(tmpdir.join('atts.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('atts.yaml'))): assert status_code == requests.get(url).status_code def test_headers(scheme, tmpdir): '''Ensure that we can read the headers back''' url = scheme + '://httpbin.org/' - with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('headers.yaml'))): headers = requests.get(url).headers - with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('headers.yaml'))): assert headers == requests.get(url).headers def test_body(tmpdir, scheme): '''Ensure the responses are all identical enough''' url = scheme + '://httpbin.org/bytes/1024' - with vcr.use_cassette(str(tmpdir.join('body.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('body.yaml'))): content = requests.get(url).content - with vcr.use_cassette(str(tmpdir.join('body.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('body.yaml'))): assert content == requests.get(url).content @@ -55,10 +55,10 @@ def test_auth(tmpdir, scheme): '''Ensure that we can handle basic auth''' auth = ('user', 'passwd') url = scheme + '://httpbin.org/basic-auth/user/passwd' - with vcr.use_cassette(str(tmpdir.join('auth.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('auth.yaml'))): one = requests.get(url, auth=auth) - with vcr.use_cassette(str(tmpdir.join('auth.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('auth.yaml'))): two = requests.get(url, auth=auth) assert one.content == two.content assert one.status_code == two.status_code @@ -68,7 +68,7 @@ def test_auth_failed(tmpdir, scheme): '''Ensure that we can save failed auth statuses''' auth = ('user', 'wrongwrongwrong') url = scheme + '://httpbin.org/basic-auth/user/passwd' - with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))): # Ensure that this is empty to begin with assert_cassette_empty(cass) one = requests.get(url, auth=auth) @@ -81,10 +81,10 @@ def test_post(tmpdir, scheme): '''Ensure that we can post and cache the results''' data = {'key1': 'value1', 'key2': 'value2'} url = scheme + '://httpbin.org/post' - with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('requests.yaml'))): req1 = requests.post(url, data).content - with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('requests.yaml'))): req2 = requests.post(url, data).content assert req1 == req2 @@ -93,7 +93,7 @@ def test_post(tmpdir, scheme): def test_redirects(tmpdir, scheme): '''Ensure that we can handle redirects''' url = scheme + '://httpbin.org/redirect-to?url=bytes/1024' - with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('requests.yaml'))): content = requests.get(url).content with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass: @@ -124,11 +124,11 @@ def test_gzip(tmpdir, scheme): url = scheme + '://httpbin.org/gzip' response = requests.get(url) - with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))): response = requests.get(url) assert_is_json(response.content) - with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))): assert_is_json(response.content) @@ -143,8 +143,8 @@ def test_session_and_connection_close(tmpdir, scheme): with vcr.use_cassette(str(tmpdir.join('session_connection_closed.yaml'))): session = requests.session() - resp = session.get('http://httpbin.org/get', headers={'Connection': 'close'}) - resp = session.get('http://httpbin.org/get', headers={'Connection': 'close'}) + session.get('http://httpbin.org/get', headers={'Connection': 'close'}) + session.get('http://httpbin.org/get', headers={'Connection': 'close'}) def test_https_with_cert_validation_disabled(tmpdir): @@ -164,9 +164,28 @@ def test_session_can_make_requests_after_requests_unpatched(tmpdir): session.get('http://httpbin.org/status/200') -def test_nested_context_managers_with_session_created_before_first_nesting(scheme, tmpdir): +def test_session_created_before_use_cassette_is_patched(tmpdir, scheme): + url = scheme + '://httpbin.org/bytes/1024' + # Record arbitrary, random data to the cassette + with vcr.use_cassette(str(tmpdir.join('session_created_outside.yaml'))): + session = requests.session() + body = session.get(url).content + + # Create a session outside of any cassette context manager + session = requests.session() + # Make a request to make sure that a connectionpool is instantiated + session.get(scheme + '://httpbin.org/get') + + with vcr.use_cassette(str(tmpdir.join('session_created_outside.yaml'))): + # These should only be the same if the patching succeeded. + assert session.get(url).content == body + + +def test_nested_cassettes_with_session_created_before_nesting(scheme, tmpdir): ''' This tests ensures that a session that was created while one cassette was + active is patched to the use the responses of a second cassette when it + is enabled. ''' url = scheme + '://httpbin.org/bytes/1024' with vcr.use_cassette(str(tmpdir.join('first_nested.yaml'))): diff --git a/tox.ini b/tox.ini index bf7a840..5797c48 100644 --- a/tox.ini +++ b/tox.ini @@ -40,220 +40,150 @@ deps = pytest pytest-localserver PyYAML + ipdb [testenv:py26requests1] basepython = python2.6 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==1.2.3 [testenv:py27requests1] basepython = python2.7 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==1.2.3 [testenv:py33requests1] basepython = python3.3 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==1.2.3 [testenv:pypyrequests1] basepython = pypy deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==1.2.3 [testenv:py26requests24] basepython = python2.6 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.4.0 [testenv:py27requests24] basepython = python2.7 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.4.0 [testenv:py33requests24] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.4.0 [testenv:py34requests24] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.4.0 [testenv:pypyrequests24] basepython = pypy deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.4.0 [testenv:py26requests23] basepython = python2.6 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.3.0 [testenv:py27requests23] basepython = python2.7 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.3.0 [testenv:py33requests23] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.3.0 [testenv:py34requests23] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.3.0 [testenv:pypyrequests23] basepython = pypy deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.3.0 [testenv:py26requests22] basepython = python2.6 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.2.1 [testenv:py27requests22] basepython = python2.7 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.2.1 [testenv:py33requests22] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.2.1 [testenv:py34requests22] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.2.1 + [testenv:pypyrequests22] basepython = pypy deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} requests==2.2.1 [testenv:py26httplib2] basepython = python2.6 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} httplib2 [testenv:py27httplib2] basepython = python2.7 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} httplib2 [testenv:py33httplib2] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} httplib2 [testenv:py34httplib2] basepython = python3.4 deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} httplib2 [testenv:pypyhttplib2] basepython = pypy deps = - mock - pytest - pytest-localserver - PyYAML + {[testenv]deps} httplib2 diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 772731e..df67b1d 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -119,7 +119,7 @@ class VCRHTTPResponse(HTTPResponse): return default -class VCRConnection: +class VCRConnection(object): # A reference to the cassette that's currently being patched in cassette = None @@ -205,7 +205,7 @@ class VCRConnection: pass def getresponse(self, _=False): - '''Retrieve a the response''' + '''Retrieve the response''' # Check to see if the cassette has a response for this request. If so, # then return it if self.cassette.can_play_response_for(self._vcr_request): From 10188678387bf385bcc89c207f55b1d5fcd4d537 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 19 Sep 2014 14:32:21 -0700 Subject: [PATCH 21/29] Revert "Fixed issue in test_nested_context_managers_with_session_created_before_first_nesting. by using a single class and patching cassette on that class. Not a great solution :\" This reverts commit 2bf23b2cdf8542f5bf137e386f3bc278bbc2cc83. --- vcr/patch.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index ab0036b..dc4e72e 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -57,22 +57,41 @@ class PatcherBuilder(object): def __init__(self, cassette): self._cassette = cassette + self._class_to_cassette_subclass = {} def build_patchers(self): patcher_args = itertools.chain(self._httplib(), self._requests(), self._urllib3(), self._httplib2(), self._boto()) - for obj, patched_attribute, replacement_class in patcher_args: - patcher = self._build_patcher(obj, patched_attribute, replacement_class) + for args in patcher_args: + patcher = self._build_patcher(*args) if patcher: yield patcher - if hasattr(replacement_class, 'cassette'): - yield mock.patch.object(replacement_class, 'cassette', self._cassette) def _build_patcher(self, obj, patched_attribute, replacement_class): if not hasattr(obj, patched_attribute): return + + if isinstance(replacement_class, dict): + for key in replacement_class: + replacement_class[key] = self._get_cassette_subclass(replacement_class[key]) + else: + replacement_class = self._get_cassette_subclass(replacement_class) return mock.patch.object(obj, patched_attribute, replacement_class) + def _get_cassette_subclass(self, klass): + if klass.cassette is not None: + return klass + if klass not in self._class_to_cassette_subclass: + self._class_to_cassette_subclass[klass] = self._build_cassette_subclass(klass) + return self._class_to_cassette_subclass[klass] + + def _build_cassette_subclass(self, base_class): + bases = (base_class,) + if not issubclass(base_class, object): # Check for old style class + bases += (object,) + return type('{0}{1}'.format(base_class.__name__, self._cassette._path), + bases, dict(cassette=self._cassette)) + def _httplib(self): yield httplib, 'HTTPConnection', VCRHTTPConnection yield httplib, 'HTTPSConnection', VCRHTTPSConnection From b1cdd50e9b633060a801e9f8669de375fbe946c9 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 19 Sep 2014 17:06:53 -0700 Subject: [PATCH 22/29] Fix some of the issues from #109 --- vcr/cassette.py | 4 +- vcr/patch.py | 129 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 110 insertions(+), 23 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index ddbb0f1..d5c33b8 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -8,7 +8,7 @@ except ImportError: from .compat.counter import Counter # Internal imports -from .patch import PatcherBuilder +from .patch import CassettePatcherBuilder from .persist import load_cassette, save_cassette from .filters import filter_request from .serializers import yamlserializer @@ -40,7 +40,7 @@ class CassetteContextDecorator(contextlib2.ContextDecorator): def _patch_generator(self, cassette): with contextlib2.ExitStack() as exit_stack: - for patcher in PatcherBuilder(cassette).build_patchers(): + for patcher in CassettePatcherBuilder(cassette).build(): exit_stack.enter_context(patcher) log.debug('Entered context for cassette at {0}.'.format(cassette._path)) yield cassette diff --git a/vcr/patch.py b/vcr/patch.py index dc4e72e..d033b55 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -1,4 +1,5 @@ '''Utilities for patching in cassettes''' +import functools import itertools import contextlib2 @@ -53,16 +54,25 @@ else: _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection -class PatcherBuilder(object): +class CassettePatcherBuilder(object): + + def _build_patchers_from_mock_triples_decorator(function): + @functools.wraps(function) + def wrapped(self, *args, **kwargs): + return self._build_patchers_from_mock_triples(function(self, *args, **kwargs)) + return wrapped def __init__(self, cassette): self._cassette = cassette self._class_to_cassette_subclass = {} - def build_patchers(self): - patcher_args = itertools.chain(self._httplib(), self._requests(), self._urllib3(), - self._httplib2(), self._boto()) - for args in patcher_args: + def build(self): + return itertools.chain(self._httplib(), self._requests(), + self._urllib3(), self._httplib2(), + self._boto()) + + def _build_patchers_from_mock_triples(self, mock_triples): + for args in mock_triples: patcher = self._build_patcher(*args) if patcher: yield patcher @@ -71,18 +81,28 @@ class PatcherBuilder(object): if not hasattr(obj, patched_attribute): return - if isinstance(replacement_class, dict): - for key in replacement_class: - replacement_class[key] = self._get_cassette_subclass(replacement_class[key]) - else: - replacement_class = self._get_cassette_subclass(replacement_class) - return mock.patch.object(obj, patched_attribute, replacement_class) + return mock.patch.object(obj, patched_attribute, + self._recursively_apply_get_cassette_subclass( + replacement_class)) + + def _recursively_apply_get_cassette_subclass(self, replacement_dict_or_obj): + if isinstance(replacement_dict_or_obj, dict): + for key, replacement_obj in replacement_dict_or_obj: + replacement_obj = self._recursively_apply_get_cassette_subclass( + replacement_obj) + replacement_dict_or_obj[key] = replacement_obj + return replacement_dict_or_obj + if hasattr(replacement_dict_or_obj, 'cassette'): + replacement_dict_or_obj = self._get_cassette_subclass( + replacement_dict_or_obj) + return replacement_dict_or_obj def _get_cassette_subclass(self, klass): if klass.cassette is not None: return klass if klass not in self._class_to_cassette_subclass: - self._class_to_cassette_subclass[klass] = self._build_cassette_subclass(klass) + subclass = self._build_cassette_subclass(klass) + self._class_to_cassette_subclass[klass] = subclass return self._class_to_cassette_subclass[klass] def _build_cassette_subclass(self, base_class): @@ -92,6 +112,7 @@ class PatcherBuilder(object): return type('{0}{1}'.format(base_class.__name__, self._cassette._path), bases, dict(cassette=self._cassette)) + @_build_patchers_from_mock_triples_decorator def _httplib(self): yield httplib, 'HTTPConnection', VCRHTTPConnection yield httplib, 'HTTPSConnection', VCRHTTPSConnection @@ -100,17 +121,51 @@ class PatcherBuilder(object): try: import requests.packages.urllib3.connectionpool as cpool except ImportError: # pragma: no cover - pass - else: - from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + return + from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + http_connection_remover = ConnectionRemover( + self._get_cassette_subclass(VCRHTTPConnection) + ) + https_connection_remover = ConnectionRemover( + self._get_cassette_subclass(VCRHTTPSConnection) + ) + mock_triples = ( + (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), + (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), + (cpool, 'HTTPConnection', VCRRequestsHTTPConnection), + (cpool, 'HTTPConnection', VCRHTTPConnection), + (cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection), + (cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection), + # These handle making sure that sessions only use the + # connections of the appropriate type. + (cpool.HTTPConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPConnectionPool)), + (cpool.HTTPSConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPSConnectionPool)), + (cpool.HTTPConnectionPool, '_new_conn', self._patched_new_conn(cpool.HTTPConnectionPool, http_connection_remover)), + (cpool.HTTPSConnectionPool, '_new_conn', self._patched_new_conn(cpool.HTTPConnectionPool, https_connection_remover)) + ) + return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), + (http_connection_remover, https_connection_remover)) - yield cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection - yield cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection - yield cpool, 'HTTPConnection', VCRRequestsHTTPConnection - yield cpool, 'HTTPConnection', VCRHTTPConnection - yield cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection - yield cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection + def _patched_get_conn(self, connection_pool_class): + get_conn = connection_pool_class._get_conn + @functools.wraps(get_conn) + def patched_get_conn(pool, timeout=None): + connection = get_conn(pool, timeout) + while not isinstance(connection, pool.ConnectionCls): + connection = get_conn(pool, timeout) + return connection + return patched_get_conn + def _patched_new_conn(self, connection_pool_class, connection_remover): + new_conn = connection_pool_class._new_conn + @functools.wraps(new_conn) + def patched_new_conn(pool): + new_connection = new_conn(pool) + connection_remover.add_connection_to_pool_entry(pool, new_connection) + return new_connection + return patched_new_conn + + @_build_patchers_from_mock_triples_decorator def _urllib3(self): try: import urllib3.connectionpool as cpool @@ -122,6 +177,7 @@ class PatcherBuilder(object): yield cpool, 'VerifiedHTTPSConnection', VCRVerifiedHTTPSConnection yield cpool, 'HTTPConnection', VCRHTTPConnection + @_build_patchers_from_mock_triples_decorator def _httplib2(self): try: import httplib2 as cpool @@ -136,6 +192,7 @@ class PatcherBuilder(object): yield cpool, 'SCHEME_TO_CONNECTION', {'http': VCRHTTPConnectionWithTimeout, 'https': VCRHTTPSConnectionWithTimeout} + @_build_patchers_from_mock_triples_decorator def _boto(self): try: import boto.https_connection as cpool @@ -146,6 +203,36 @@ class PatcherBuilder(object): yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection +class ConnectionRemover(object): + + def __init__(self, connection_class): + self._connection_class = connection_class + self._connection_pool_to_connections = {} + + def add_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connection.setdefault(pool, set()).add(connection) + + def remove_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connection[self._connection_class].remove(connection) + + def __enter__(self): + return self + + def __exit__(self, *args): + for pool, connections in self._connection_pool_to_connections.items(): + readd_connections = [] + while pool.not_empty() and connections: + connection = pool.get() + if isinstance(connection, self._connection_class): + connections.remove(connection) + else: + readd_connections.append(connection) + for connection in readd_connections: + self.pool._put_conn(connection) + + def reset_patchers(): yield mock.patch.object(httplib, 'HTTPConnection', _HTTPConnection) yield mock.patch.object(httplib, 'HTTPSConnection', _HTTPSConnection) From 121ed7917213baf300a274c097281580cbb3f12c Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 19 Sep 2014 17:10:19 -0700 Subject: [PATCH 23/29] Mark bad test xfail. --- tests/integration/test_requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index 9062d7d..f7bf7ff 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -68,7 +68,7 @@ def test_auth_failed(tmpdir, scheme): '''Ensure that we can save failed auth statuses''' auth = ('user', 'wrongwrongwrong') url = scheme + '://httpbin.org/basic-auth/user/passwd' - with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))): + with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))) as cass: # Ensure that this is empty to begin with assert_cassette_empty(cass) one = requests.get(url, auth=auth) @@ -151,7 +151,7 @@ def test_https_with_cert_validation_disabled(tmpdir): with vcr.use_cassette(str(tmpdir.join('cert_validation_disabled.yaml'))): requests.get('https://httpbin.org', verify=False) - +@pytest.mark.xfail def test_session_can_make_requests_after_requests_unpatched(tmpdir): with vcr.use_cassette(str(tmpdir.join('test_session_after_unpatched.yaml'))): session = requests.session() From dc249b09656acb27651e04071592e276ecf4dca8 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 19 Sep 2014 17:20:59 -0700 Subject: [PATCH 24/29] Remove ConnectionRemover class that tried to get rid of vcr connections in ConnectionPools. --- vcr/patch.py | 50 +------------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index d033b55..a8c5aaa 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -123,12 +123,6 @@ class CassettePatcherBuilder(object): except ImportError: # pragma: no cover return from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection - http_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRHTTPConnection) - ) - https_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRHTTPSConnection) - ) mock_triples = ( (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), @@ -140,11 +134,8 @@ class CassettePatcherBuilder(object): # connections of the appropriate type. (cpool.HTTPConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPConnectionPool)), (cpool.HTTPSConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPSConnectionPool)), - (cpool.HTTPConnectionPool, '_new_conn', self._patched_new_conn(cpool.HTTPConnectionPool, http_connection_remover)), - (cpool.HTTPSConnectionPool, '_new_conn', self._patched_new_conn(cpool.HTTPConnectionPool, https_connection_remover)) ) - return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), - (http_connection_remover, https_connection_remover)) + return self._build_patchers_from_mock_triples(mock_triples) def _patched_get_conn(self, connection_pool_class): get_conn = connection_pool_class._get_conn @@ -156,15 +147,6 @@ class CassettePatcherBuilder(object): return connection return patched_get_conn - def _patched_new_conn(self, connection_pool_class, connection_remover): - new_conn = connection_pool_class._new_conn - @functools.wraps(new_conn) - def patched_new_conn(pool): - new_connection = new_conn(pool) - connection_remover.add_connection_to_pool_entry(pool, new_connection) - return new_connection - return patched_new_conn - @_build_patchers_from_mock_triples_decorator def _urllib3(self): try: @@ -203,36 +185,6 @@ class CassettePatcherBuilder(object): yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection -class ConnectionRemover(object): - - def __init__(self, connection_class): - self._connection_class = connection_class - self._connection_pool_to_connections = {} - - def add_connection_to_pool_entry(self, pool, connection): - if isinstance(connection, self._connection_class): - self._connection_pool_to_connection.setdefault(pool, set()).add(connection) - - def remove_connection_to_pool_entry(self, pool, connection): - if isinstance(connection, self._connection_class): - self._connection_pool_to_connection[self._connection_class].remove(connection) - - def __enter__(self): - return self - - def __exit__(self, *args): - for pool, connections in self._connection_pool_to_connections.items(): - readd_connections = [] - while pool.not_empty() and connections: - connection = pool.get() - if isinstance(connection, self._connection_class): - connections.remove(connection) - else: - readd_connections.append(connection) - for connection in readd_connections: - self.pool._put_conn(connection) - - def reset_patchers(): yield mock.patch.object(httplib, 'HTTPConnection', _HTTPConnection) yield mock.patch.object(httplib, 'HTTPSConnection', _HTTPSConnection) From 83211a18873ab81691ab69980630c3994124b46d Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sat, 20 Sep 2014 10:55:03 -0700 Subject: [PATCH 25/29] Make changes from b1cdd50e9b633060a801e9f8669de375fbe946c9 compatible with requests1.x; Update Readme.md with description of `before_record_response` --- README.md | 64 ++++++++++++++++++++++++++++++++++++++++------------ vcr/patch.py | 16 +++++++++---- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0a9c954..5c55b00 100644 --- a/README.md +++ b/README.md @@ -176,13 +176,13 @@ with vcr.use_cassette('fixtures/vcr_cassettes/synopsis.yaml') as cass: The `Cassette` object exposes the following properties which I consider part of the API. The fields are as follows: - * `requests`: A list of vcr.Request objects containing the requests made while - this cassette was being used, ordered by the order that the request was made. + * `requests`: A list of vcr.Request objects corresponding to the http requests + that were made during the recording of the cassette. The requests appear in the + order that they were originally processed. * `responses`: A list of the responses made. - * `play_count`: The number of times this cassette has had a response played - back - * `all_played`: A boolean indicates whether all the responses have been - played back + * `play_count`: The number of times this cassette has played back a response. + * `all_played`: A boolean indicating whether all the responses have been + played back. * `responses_of(request)`: Access the responses that match a given request The `Request` object has the following properties: @@ -215,7 +215,7 @@ Finally, register your class with VCR to use your new serializer. ```python import vcr -BogoSerializer(object): +class BogoSerializer(object): """ Must implement serialize() and deserialize() methods """ @@ -293,12 +293,12 @@ with my_vcr.use_cassette('test.yml', filter_query_parameters=['api_key']): requests.get('http://api.com/getdata?api_key=secretstring') ``` -### Custom request filtering +### Custom Request filtering -If neither of these covers your use case, you can register a callback that will -manipulate the HTTP request before adding it to the cassette. Use the -`before_record` configuration option to so this. Here is an -example that will never record requests to the /login endpoint. +If neither of these covers your request filtering needs, you can register a callback +that will manipulate the HTTP request before adding it to the cassette. Use the +`before_record` configuration option to so this. Here is an example that will + never record requests to the /login endpoint. ```python def before_record_cb(request): @@ -312,6 +312,40 @@ with my_vcr.use_cassette('test.yml'): # your http code here ``` +You can also mutate the response using this callback. For example, you could +remove all query parameters from any requests to the `'/login'` path. + +```python +def scrub_login_request(request): + if request.path == '/login': + request.uri, _ = urllib.splitquery(response.uri) + return request + +my_vcr = vcr.VCR( + before_record=scrub_login_request, +) +with my_vcr.use_cassette('test.yml'): + # your http code here +``` + +### Custom Response Filtering + +VCR.py also suports response filtering with the `before_record_response` keyword +argument. It's usage is similar to that of `before_record`: + +```python +def scrub_string(string, replacement=''): + def before_record_reponse(response): + return response['body']['string] = response['body']['string].replace(string, replacement) + return scrub_string + +my_vcr = vcr.VCR( + before_record=scrub_string(settings.USERNAME, 'username'), +) +with my_vcr.use_cassette('test.yml'): + # your http code here +``` + ## Ignore requests If you would like to completely ignore certain requests, you can do it in a @@ -335,7 +369,7 @@ to `brew install libyaml` [[Homebrew](http://mxcl.github.com/homebrew/)]) ## Ruby VCR compatibility -I'm not trying to match the format of the Ruby VCR YAML files. Cassettes +VCR.py does not aim to match the format of the Ruby VCR YAML files. Cassettes generated by Ruby's VCR are not compatible with VCR.py. ## Running VCR's test suite @@ -356,7 +390,7 @@ installed. Also, in order for the boto tests to run, you will need an AWS key. Refer to the [boto documentation](http://boto.readthedocs.org/en/latest/getting_started.html) for -how to set this up. I have marked the boto tests as optional in Travis so you +how to set this up. I have marked the boto tests as optional in Travis so you don't have to worry about them failing if you submit a pull request. @@ -423,6 +457,8 @@ API in version 1.0.x ## Changelog + * 1.1.0 Add `before_record_response`. Fix several bugs related to the context + management of cassettes. * 1.0.3: Fix an issue with requests 2.4 and make sure case sensitivity is consistent across python versions * 1.0.2: Fix an issue with requests 2.3 diff --git a/vcr/patch.py b/vcr/patch.py index a8c5aaa..a3672f4 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -127,22 +127,28 @@ class CassettePatcherBuilder(object): (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), (cpool, 'HTTPConnection', VCRRequestsHTTPConnection), - (cpool, 'HTTPConnection', VCRHTTPConnection), + (cpool, 'HTTPSConnection', VCRRequestsHTTPSConnection), (cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection), (cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection), # These handle making sure that sessions only use the # connections of the appropriate type. - (cpool.HTTPConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPConnectionPool)), - (cpool.HTTPSConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPSConnectionPool)), ) + mock_triples += ((cpool.HTTPConnectionPool, '_get_conn', + self._patched_get_conn(cpool.HTTPConnectionPool, + lambda : cpool.HTTPConnection)), + (cpool.HTTPSConnectionPool, '_get_conn', + self._patched_get_conn(cpool.HTTPSConnectionPool, + lambda : cpool.HTTPSConnection))) return self._build_patchers_from_mock_triples(mock_triples) - def _patched_get_conn(self, connection_pool_class): + def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @functools.wraps(get_conn) def patched_get_conn(pool, timeout=None): connection = get_conn(pool, timeout) - while not isinstance(connection, pool.ConnectionCls): + connection_class = pool.ConnectionCls if hasattr(pool, 'ConnectionCls') \ + else connection_class_getter() + while not isinstance(connection, connection_class): connection = get_conn(pool, timeout) return connection return patched_get_conn From 18e5898ec420d7e5b29e53c12abf475ca6ed829a Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sat, 20 Sep 2014 11:28:59 -0700 Subject: [PATCH 26/29] Return a tuple from the _request function on CassettePatcherBuilder even if import fails. Make _recursively_apply_get_cassette_subclass actually work with dictionaries. --- vcr/patch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index a3672f4..74bb355 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -87,7 +87,7 @@ class CassettePatcherBuilder(object): def _recursively_apply_get_cassette_subclass(self, replacement_dict_or_obj): if isinstance(replacement_dict_or_obj, dict): - for key, replacement_obj in replacement_dict_or_obj: + for key, replacement_obj in replacement_dict_or_obj.items(): replacement_obj = self._recursively_apply_get_cassette_subclass( replacement_obj) replacement_dict_or_obj[key] = replacement_obj @@ -121,7 +121,7 @@ class CassettePatcherBuilder(object): try: import requests.packages.urllib3.connectionpool as cpool except ImportError: # pragma: no cover - return + return () from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection mock_triples = ( (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), From 757ad9c8362aa233ce753de604218cba5902feea Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sat, 20 Sep 2014 11:59:25 -0700 Subject: [PATCH 27/29] Revert "Remove ConnectionRemover class that tried to get rid of vcr connections in ConnectionPools." This reverts commit dc249b09656acb27651e04071592e276ecf4dca8. Conflicts: vcr/patch.py --- vcr/patch.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/vcr/patch.py b/vcr/patch.py index 74bb355..1a09801 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -123,6 +123,12 @@ class CassettePatcherBuilder(object): except ImportError: # pragma: no cover return () from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + http_connection_remover = ConnectionRemover( + self._get_cassette_subclass(VCRHTTPConnection) + ) + https_connection_remover = ConnectionRemover( + self._get_cassette_subclass(VCRHTTPSConnection) + ) mock_triples = ( (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), @@ -130,16 +136,24 @@ class CassettePatcherBuilder(object): (cpool, 'HTTPSConnection', VCRRequestsHTTPSConnection), (cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection), (cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection), - # These handle making sure that sessions only use the - # connections of the appropriate type. ) + # These handle making sure that sessions only use the + # connections of the appropriate type. mock_triples += ((cpool.HTTPConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPConnectionPool, lambda : cpool.HTTPConnection)), (cpool.HTTPSConnectionPool, '_get_conn', self._patched_get_conn(cpool.HTTPSConnectionPool, - lambda : cpool.HTTPSConnection))) - return self._build_patchers_from_mock_triples(mock_triples) + lambda : cpool.HTTPSConnection)), + (cpool.HTTPConnectionPool, '_new_conn', + self._patched_new_conn(cpool.HTTPConnectionPool, + http_connection_remover)), + (cpool.HTTPSConnectionPool, '_new_conn', + self._patched_new_conn(cpool.HTTPConnectionPool, + https_connection_remover))) + + return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), + (http_connection_remover, https_connection_remover)) def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @@ -153,6 +167,15 @@ class CassettePatcherBuilder(object): return connection return patched_get_conn + def _patched_new_conn(self, connection_pool_class, connection_remover): + new_conn = connection_pool_class._new_conn + @functools.wraps(new_conn) + def patched_new_conn(pool): + new_connection = new_conn(pool) + connection_remover.add_connection_to_pool_entry(pool, new_connection) + return new_connection + return patched_new_conn + @_build_patchers_from_mock_triples_decorator def _urllib3(self): try: @@ -191,6 +214,36 @@ class CassettePatcherBuilder(object): yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection +class ConnectionRemover(object): + + def __init__(self, connection_class): + self._connection_class = connection_class + self._connection_pool_to_connections = {} + + def add_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connection.setdefault(pool, set()).add(connection) + + def remove_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connection[self._connection_class].remove(connection) + + def __enter__(self): + return self + + def __exit__(self, *args): + for pool, connections in self._connection_pool_to_connections.items(): + readd_connections = [] + while pool.not_empty() and connections: + connection = pool.get() + if isinstance(connection, self._connection_class): + connections.remove(connection) + else: + readd_connections.append(connection) + for connection in readd_connections: + self.pool._put_conn(connection) + + def reset_patchers(): yield mock.patch.object(httplib, 'HTTPConnection', _HTTPConnection) yield mock.patch.object(httplib, 'HTTPSConnection', _HTTPSConnection) From a2c947dc4822a518f79815f7b78e938874e571b0 Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sun, 21 Sep 2014 05:06:28 -0700 Subject: [PATCH 28/29] Fix last bit of of #109. --- tests/integration/test_requests.py | 2 +- vcr/patch.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index f7bf7ff..9f6484f 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -151,7 +151,7 @@ def test_https_with_cert_validation_disabled(tmpdir): with vcr.use_cassette(str(tmpdir.join('cert_validation_disabled.yaml'))): requests.get('https://httpbin.org', verify=False) -@pytest.mark.xfail + def test_session_can_make_requests_after_requests_unpatched(tmpdir): with vcr.use_cassette(str(tmpdir.join('test_session_after_unpatched.yaml'))): session = requests.session() diff --git a/vcr/patch.py b/vcr/patch.py index 1a09801..6ab8627 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -124,10 +124,10 @@ class CassettePatcherBuilder(object): return () from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection http_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRHTTPConnection) + self._get_cassette_subclass(VCRRequestsHTTPConnection) ) https_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRHTTPSConnection) + self._get_cassette_subclass(VCRRequestsHTTPSConnection) ) mock_triples = ( (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), @@ -149,7 +149,7 @@ class CassettePatcherBuilder(object): self._patched_new_conn(cpool.HTTPConnectionPool, http_connection_remover)), (cpool.HTTPSConnectionPool, '_new_conn', - self._patched_new_conn(cpool.HTTPConnectionPool, + self._patched_new_conn(cpool.HTTPSConnectionPool, https_connection_remover))) return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), @@ -222,11 +222,11 @@ class ConnectionRemover(object): def add_connection_to_pool_entry(self, pool, connection): if isinstance(connection, self._connection_class): - self._connection_pool_to_connection.setdefault(pool, set()).add(connection) + self._connection_pool_to_connections.setdefault(pool, set()).add(connection) def remove_connection_to_pool_entry(self, pool, connection): if isinstance(connection, self._connection_class): - self._connection_pool_to_connection[self._connection_class].remove(connection) + self._connection_pool_to_connections[self._connection_class].remove(connection) def __enter__(self): return self @@ -234,14 +234,14 @@ class ConnectionRemover(object): def __exit__(self, *args): for pool, connections in self._connection_pool_to_connections.items(): readd_connections = [] - while pool.not_empty() and connections: - connection = pool.get() + while not pool.pool.empty() and connections: + connection = pool.pool.get() if isinstance(connection, self._connection_class): connections.remove(connection) else: readd_connections.append(connection) for connection in readd_connections: - self.pool._put_conn(connection) + pool._put_conn(connection) def reset_patchers(): From 113c95f9711218192ff57e0acb15e176f5c3134d Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sun, 21 Sep 2014 05:07:24 -0700 Subject: [PATCH 29/29] Bump setup.py version to v1.1.0. minor tweaks to readme. --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c55b00..7ed85af 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![vcr.py](https://raw.github.com/kevin1024/vcrpy/master/vcr.png) -This is a Python version of [Ruby's VCR library](https://github.com/myronmarston/vcr). +This is a Python version of [Ruby's VCR library](https://github.com/vcr/vcr). [![Build Status](https://secure.travis-ci.org/kevin1024/vcrpy.png?branch=master)](http://travis-ci.org/kevin1024/vcrpy) [![Stories in Ready](https://badge.waffle.io/kevin1024/vcrpy.png?label=ready&title=Ready)](https://waffle.io/kevin1024/vcrpy) diff --git a/setup.py b/setup.py index 3e1fc05..df31370 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ class PyTest(TestCommand): setup( name='vcrpy', - version='1.0.4', + version='1.1.0', description=( "Automatically mock your HTTP interactions to simplify and " "speed up testing"