v1.7.3

Trivial cleanups and one bugfix

README.rst

@@ -14,17 +14,18 @@ library <https://github.com/vcr/vcr>`__.
 What it does
 ------------
 
-VCR.py simplifies and speeds up tests that make HTTP requests. The first
-time you run code that is inside a VCR.py context manager or decorated
-function, VCR.py records all HTTP interactions that take place through
-the libraries it supports and serializes and writes them to a flat file
-(in yaml format by default). This flat file is called a cassette. When
-the relevant peice of code is executed again, VCR.py will read the
-serialized requests and responses from the aforementioned cassette file,
-and intercept any HTTP requests that it recognizes from the original
-test run and return responses that corresponded to those requests. This
-means that the requests will not actually result in HTTP traffic, which
-confers several benefits including:
+VCR.py simplifies and speeds up tests that make HTTP requests. The
+first time you run code that is inside a VCR.py context manager or
+decorated function, VCR.py records all HTTP interactions that take
+place through the libraries it supports and serializes and writes them
+to a flat file (in yaml format by default). This flat file is called a
+cassette. When the relevant peice of code is executed again, VCR.py
+will read the serialized requests and responses from the
+aforementioned cassette file, and intercept any HTTP requests that it
+recognizes from the original test run and return the responses that
+corresponded to those requests. This means that the requests will not
+actually result in HTTP traffic, which confers several benefits
+including:
 
 -  The ability to work offline
 -  Completely deterministic tests
@@ -608,6 +609,11 @@ new API in version 1.0.x
 
 Changelog
 ---------
+-  1.7.3 [#188] ``additional_matchers`` kwarg on ``use_casstte``.
+   [#191] Actually support passing multiple before_record_request
+   functions (thanks @agriffis).
+-  1.7.2 [#186] Get effective_url in tornado (thanks @mvschaik), [#187]
+   Set request_time on Response object in tornado (thanks @abhinav).
 -  1.7.1 [#183] Patch ``fetch_impl`` instead of the entire HTTPClient
    class for Tornado (thanks @abhinav).
 -  1.7.0 [#177] Properly support coroutine/generator decoration. [#178]

setup.py

@@ -51,7 +51,7 @@ except Exception:
 
 setup(
     name='vcrpy',
-    version='1.7.1',
+    version='1.7.3',
     description=(
         "Automatically mock your HTTP interactions to simplify and "
         "speed up testing"

tests/integration/test_httplib2.py

@@ -56,6 +56,17 @@ def test_response_headers(scheme, tmpdir):
         resp, _ = httplib2.Http().request(url)
         assert set(headers) == set(resp.items())
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        resp, _ = httplib2.Http().request(url)
+        effective_url = resp['content-location']
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        resp, _ = httplib2.Http().request(url)
+        assert effective_url == resp['content-location']
 
 def test_multiple_requests(scheme, tmpdir):
     '''Ensure that we can cache multiple requests'''

tests/integration/test_requests.py

@@ -44,6 +44,15 @@ def test_body(tmpdir, scheme):
     with vcr.use_cassette(str(tmpdir.join('body.yaml'))):
         assert content == requests.get(url).content
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        effective_url = requests.get(url).url
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        assert effective_url == requests.get(url).url
 
 def test_auth(tmpdir, scheme):
     '''Ensure that we can handle basic auth'''

tests/integration/test_tornado.py

@@ -81,6 +81,17 @@ def test_body(get_client, tmpdir, scheme):
         assert content == (yield get(get_client(), url)).body
         assert 1 == cass.play_count
 
+@pytest.mark.gen_test
+def test_effective_url(get_client, scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        effective_url = (yield get(get_client(), url)).effective_url
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))) as cass:
+        assert effective_url == (yield get(get_client(), url)).effective_url
+        assert 1 == cass.play_count
 
 @pytest.mark.gen_test
 def test_auth(get_client, tmpdir, scheme):
@@ -326,3 +337,19 @@ def test_existing_instances_get_patched(get_client, tmpdir):
     with vcr.use_cassette(str(tmpdir.join('data.yaml'))) as cass:
         yield get(client, 'http://httpbin.org/get')
         assert cass.play_count == 1
+
+
+@pytest.mark.gen_test
+def test_request_time_is_set(get_client, tmpdir):
+    '''Ensures that the request_time on HTTPResponses is set.'''
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))):
+        client = get_client()
+        response = yield get(client, 'http://httpbin.org/get')
+        assert response.request_time is not None
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))) as cass:
+        client = get_client()
+        response = yield get(client, 'http://httpbin.org/get')
+        assert response.request_time is not None
+        assert cass.play_count == 1

tests/integration/test_urllib2.py

@@ -49,6 +49,15 @@ def test_response_headers(scheme, tmpdir):
         open2 = urlopen(url).info().items()
         assert sorted(open1) == sorted(open2)
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        effective_url = urlopen(url).geturl()
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        assert effective_url == urlopen(url).geturl()
 
 def test_multiple_requests(scheme, tmpdir):
     '''Ensure that we can cache multiple requests'''

tests/unit/test_vcr.py

@@ -15,9 +15,11 @@ def test_vcr_use_cassette():
         'vcr.cassette.Cassette.load',
         return_value=mock.MagicMock(inject=False)
     ) as mock_cassette_load:
+
         @test_vcr.use_cassette('test')
         def function():
             pass
+
         assert mock_cassette_load.call_count == 0
         function()
         assert mock_cassette_load.call_args[1]['record_mode'] is record_mode
@@ -38,9 +40,11 @@ def test_vcr_use_cassette():
 
 def test_vcr_before_record_request_params():
     base_path = 'http://httpbin.org/'
+
     def before_record_cb(request):
         if request.path != '/get':
             return request
+
     test_vcr = VCR(filter_headers=('cookie',), before_record_request=before_record_cb,
                    ignore_hosts=('www.test.com',), ignore_localhost=True,
                    filter_query_parameters=('foo',))
@@ -53,8 +57,12 @@ def test_vcr_before_record_request_params():
         assert cassette.filter_request(
             Request('GET', base_path + '?foo=bar', '',
                     {'cookie': 'test', 'other': 'fun'})).headers == {'other': 'fun'}
-        assert cassette.filter_request(Request('GET', base_path + '?foo=bar', '',
-                                               {'cookie': 'test', 'other': 'fun'})).headers == {'other': 'fun'}
+        assert cassette.filter_request(
+            Request(
+                'GET', base_path + '?foo=bar', '',
+                {'cookie': 'test', 'other': 'fun'}
+            )
+        ).headers == {'other': 'fun'}
 
         assert cassette.filter_request(Request('GET', 'http://www.test.com' + '?foo=bar', '',
                                                {'cookie': 'test', 'other': 'fun'})) is None
@@ -64,6 +72,32 @@ def test_vcr_before_record_request_params():
         assert cassette.filter_request(Request('GET', base_path + 'get', '', {})) is not None
 
 
+def test_vcr_before_record_response_iterable():
+    # Regression test for #191
+
+    request = Request('GET', '/', '', {})
+    response = object()  # just can't be None
+
+    # Prevent actually saving the cassette
+    with mock.patch('vcr.cassette.save_cassette'):
+
+        # Baseline: non-iterable before_record_response should work
+        mock_filter = mock.Mock()
+        vcr = VCR(before_record_response=mock_filter)
+        with vcr.use_cassette('test') as cassette:
+            assert mock_filter.call_count == 0
+            cassette.append(request, response)
+            assert mock_filter.call_count == 1
+
+        # Regression test: iterable before_record_response should work too
+        mock_filter = mock.Mock()
+        vcr = VCR(before_record_response=(mock_filter,))
+        with vcr.use_cassette('test') as cassette:
+            assert mock_filter.call_count == 0
+            cassette.append(request, response)
+            assert mock_filter.call_count == 1
+
+
 @pytest.fixture
 def random_fixture():
     return 1
@@ -103,6 +137,7 @@ def test_custom_patchers():
 
 def test_inject_cassette():
     vcr = VCR(inject_cassette=True)
+
     @vcr.use_cassette('test', record_mode='once')
     def with_cassette_injected(cassette):
         assert cassette.record_mode == 'once'
@@ -117,9 +152,11 @@ def test_inject_cassette():
 
 def test_with_current_defaults():
     vcr = VCR(inject_cassette=True, record_mode='once')
+
     @vcr.use_cassette('test', with_current_defaults=False)
     def changing_defaults(cassette, checks):
         checks(cassette)
+
     @vcr.use_cassette('test', with_current_defaults=True)
     def current_defaults(cassette, checks):
         checks(cassette)
@@ -141,27 +178,33 @@ def test_with_current_defaults():
 def test_cassette_library_dir_with_decoration_and_no_explicit_path():
     library_dir = '/libary_dir'
     vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir)
+
     @vcr.use_cassette()
     def function_name(cassette):
         assert cassette._path == os.path.join(library_dir, 'function_name')
+
     function_name()
 
 
 def test_cassette_library_dir_with_decoration_and_explicit_path():
     library_dir = '/libary_dir'
     vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir)
+
     @vcr.use_cassette(path='custom_name')
     def function_name(cassette):
         assert cassette._path == os.path.join(library_dir, 'custom_name')
+
     function_name()
 
 
 def test_cassette_library_dir_with_decoration_and_super_explicit_path():
     library_dir = '/libary_dir'
     vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir)
+
     @vcr.use_cassette(path=os.path.join(library_dir, 'custom_name'))
     def function_name(cassette):
         assert cassette._path == os.path.join(library_dir, 'custom_name')
+
     function_name()
 
 
@@ -169,26 +212,32 @@ def test_cassette_library_dir_with_path_transformer():
     library_dir = '/libary_dir'
     vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir,
               path_transformer=lambda path: path + '.json')
+
     @vcr.use_cassette()
     def function_name(cassette):
         assert cassette._path == os.path.join(library_dir, 'function_name.json')
+
     function_name()
 
 
 def test_use_cassette_with_no_extra_invocation():
     vcr = VCR(inject_cassette=True, cassette_library_dir='/')
+
     @vcr.use_cassette
     def function_name(cassette):
         assert cassette._path == os.path.join('/', 'function_name')
+
     function_name()
 
 
 def test_path_transformer():
     vcr = VCR(inject_cassette=True, cassette_library_dir='/',
               path_transformer=lambda x: x + '_test')
+
     @vcr.use_cassette
     def function_name(cassette):
         assert cassette._path == os.path.join('/', 'function_name_test')
+
     function_name()
 
 
@@ -203,8 +252,25 @@ def test_cassette_name_generator_defaults_to_using_module_function_defined_in():
 
 def test_ensure_suffix():
     vcr = VCR(inject_cassette=True, path_transformer=VCR.ensure_suffix('.yaml'))
+
     @vcr.use_cassette
     def function_name(cassette):
         assert cassette._path == os.path.join(os.path.dirname(__file__),
                                               'function_name.yaml')
+
     function_name()
+
+
+def test_additional_matchers():
+    vcr = VCR(match_on=('uri',), inject_cassette=True)
+
+    @vcr.use_cassette
+    def function_defaults(cassette):
+        assert set(cassette._match_on) == set([vcr.matchers['uri']])
+
+    @vcr.use_cassette(additional_matchers=('body',))
+    def function_additional(cassette):
+        assert set(cassette._match_on) == set([vcr.matchers['uri'], vcr.matchers['body']])
+
+    function_defaults()
+    function_additional()

vcr/cassette.py

@@ -20,10 +20,18 @@ class CassetteContextDecorator(object):
     """Context manager/decorator that handles installing the cassette and
     removing cassettes.
 
-    This class defers the creation of a new cassette instance until the point at
-    which it is installed by context manager or decorator. The fact that a new
-    cassette is used with each application prevents the state of any cassette
-    from interfering with another.
+    This class defers the creation of a new cassette instance until
+    the point at which it is installed by context manager or
+    decorator. The fact that a new cassette is used with each
+    application prevents the state of any cassette from interfering
+    with another.
+
+    Instances of this class are NOT reentrant as context managers.
+    However, functions that are decorated by
+    ``CassetteContextDecorator`` instances ARE reentrant. See the
+    implementation of ``__call__`` on this class for more details.
+    There is also a guard against attempts to reenter instances of
+    this class as a context manager in ``__exit__``.
     """
 
     _non_cassette_arguments = ('path_transformer', 'func_path_generator')
@@ -41,9 +49,14 @@ class CassetteContextDecorator(object):
         with contextlib.ExitStack() as exit_stack:
             for patcher in CassettePatcherBuilder(cassette).build():
                 exit_stack.enter_context(patcher)
-            log.debug('Entered context for cassette at {0}.'.format(cassette._path))
+            log_format = '{action} context for cassette at {path}.'
+            log.debug(log_format.format(
+                action="Entering", path=cassette._path
+            ))
             yield cassette
-            log.debug('Exiting context for cassette at {0}.'.format(cassette._path))
+            log.debug(log_format.format(
+                action="Exiting", path=cassette._path
+            ))
             # TODO(@IvanMalison): Hmmm. it kind of feels like this should be
             # somewhere else.
             cassette._save()
@@ -78,7 +91,9 @@ class CassetteContextDecorator(object):
         # functions are reentrant. This is required for thread
         # safety and the correct operation of recursive functions.
         args_getter = self._build_args_getter_for_decorator(function)
-        return type(self)(self.cls, args_getter)._execute_function(function, args, kwargs)
+        return type(self)(self.cls, args_getter)._execute_function(
+            function, args, kwargs
+        )
 
     def _execute_function(self, function, args, kwargs):
         if inspect.isgeneratorfunction(function):
@@ -195,7 +210,6 @@ class Cassette(object):
         request = self._before_record_request(request)
         if not request:
             return
-        if self._before_record_response:
         response = self._before_record_response(response)
         self.data.append((request, response))
         self.dirty = True

vcr/config.py

@@ -67,10 +67,11 @@ class VCR(object):
         try:
             serializer = self.serializers[serializer_name]
         except KeyError:
-            print("Serializer {0} doesn't exist or isn't registered".format(
+            raise KeyError(
+                "Serializer {0} doesn't exist or isn't registered".format(
                     serializer_name
-            ))
-            raise KeyError
+                )
+            )
         return serializer
 
     def _get_matchers(self, matcher_names):
@@ -117,12 +118,16 @@ class VCR(object):
             'cassette_library_dir',
             self.cassette_library_dir
         )
+        additional_matchers = kwargs.get('additional_matchers', ())
+
         if cassette_library_dir:
             def add_cassette_library_dir(path):
                 if not path.startswith(cassette_library_dir):
                     return os.path.join(cassette_library_dir, path)
                 return path
-            path_transformer = compose(add_cassette_library_dir, path_transformer)
+            path_transformer = compose(
+                add_cassette_library_dir, path_transformer
+            )
         elif not func_path_generator:
             # If we don't have a library dir, use the functions
             # location to build a full path for cassettes.
@@ -130,12 +135,12 @@ class VCR(object):
 
         merged_config = {
             'serializer': self._get_serializer(serializer_name),
-            'match_on': self._get_matchers(matcher_names),
+            'match_on': self._get_matchers(
+                tuple(matcher_names) + tuple(additional_matchers)
+            ),
             'record_mode': kwargs.get('record_mode', self.record_mode),
             'before_record_request': self._build_before_record_request(kwargs),
-            'before_record_response': self._build_before_record_response(
-                kwargs
-            ),
+            'before_record_response': self._build_before_record_response(kwargs),
             'custom_patches': self._custom_patches + kwargs.get(
                 'custom_patches', ()
             ),
@@ -153,11 +158,11 @@ class VCR(object):
             'before_record_response', self.before_record_response
         )
         filter_functions = []
-        if before_record_response and not isinstance(before_record_response,
-                                                     collections.Iterable):
+        if before_record_response:
+            if not isinstance(before_record_response, collections.Iterable):
                 before_record_response = (before_record_response,)
-            for function in before_record_response:
-                filter_functions.append(function)
+            filter_functions.extend(before_record_response)
+
         def before_record_response(response):
             for function in filter_functions:
                 if response is None:
@@ -178,7 +183,8 @@ class VCR(object):
             'filter_post_data_parameters', self.filter_post_data_parameters
         )
         before_record_request = options.get(
-            "before_record_request", options.get("before_record", self.before_record_request)
+            "before_record_request",
+            options.get("before_record", self.before_record_request)
         )
         ignore_hosts = options.get(
             'ignore_hosts', self.ignore_hosts
@@ -187,28 +193,36 @@ class VCR(object):
             'ignore_localhost', self.ignore_localhost
         )
         if filter_headers:
-            filter_functions.append(functools.partial(filters.remove_headers,
-                                                      headers_to_remove=filter_headers))
+            filter_functions.append(
+                functools.partial(
+                    filters.remove_headers,
+                    headers_to_remove=filter_headers
+                )
+            )
         if filter_query_parameters:
-            filter_functions.append(functools.partial(filters.remove_query_parameters,
-                                                      query_parameters_to_remove=filter_query_parameters))
+            filter_functions.append(functools.partial(
+                filters.remove_query_parameters,
+                query_parameters_to_remove=filter_query_parameters
+            ))
         if filter_post_data_parameters:
-            filter_functions.append(functools.partial(filters.remove_post_data_parameters,
-                                                      post_data_parameters_to_remove=filter_post_data_parameters))
+            filter_functions.append(
+                functools.partial(
+                    filters.remove_post_data_parameters,
+                    post_data_parameters_to_remove=filter_post_data_parameters
+                )
+            )
 
-        hosts_to_ignore = list(ignore_hosts)
+        hosts_to_ignore = set(ignore_hosts)
         if ignore_localhost:
-            hosts_to_ignore.extend(('localhost', '0.0.0.0', '127.0.0.1'))
-
+            hosts_to_ignore.update(('localhost', '0.0.0.0', '127.0.0.1'))
         if hosts_to_ignore:
-            hosts_to_ignore = set(hosts_to_ignore)
             filter_functions.append(self._build_ignore_hosts(hosts_to_ignore))
 
         if before_record_request:
             if not isinstance(before_record_request, collections.Iterable):
                 before_record_request = (before_record_request,)
-            for function in before_record_request:
-                filter_functions.append(function)
+            filter_functions.extend(before_record_request)
+
         def before_record_request(request):
             request = copy.copy(request)
             for function in filter_functions:
@@ -216,7 +230,6 @@ class VCR(object):
                     break
                 request = function(request)
             return request
-
         return before_record_request
 
     @staticmethod

vcr/errors.py

@@ -3,8 +3,5 @@ class CannotOverwriteExistingCassetteException(Exception):
 
 
 class UnhandledHTTPRequestError(KeyError):
-    '''
-    Raised when a cassette does not c
-    ontain the request we want
-    '''
+    """Raised when a cassette does not contain the request we want."""
     pass

vcr/matchers.py

@@ -2,6 +2,8 @@ import json
 from six.moves import urllib, xmlrpc_client
 from .util import CaseInsensitiveDict, read_body
 import logging
+
+
 log = logging.getLogger(__name__)
 
 

vcr/stubs/tornado_stubs.py

@@ -36,6 +36,7 @@ def vcr_fetch_impl(cassette, real_fetch_impl):
                     "that is not yet supported by VCR.py. Please make the "
                     "request outside a VCR.py context." % repr(request)
                 ),
+                request_time=self.io_loop.time() - request.start_time,
             )
             return callback(response)
 
@@ -62,6 +63,8 @@ def vcr_fetch_impl(cassette, real_fetch_impl):
                 reason=vcr_response['status']['message'],
                 headers=headers,
                 buffer=BytesIO(vcr_response['body']['string']),
+                effective_url=vcr_response.get('url'),
+                request_time=self.io_loop.time() - request.start_time,
             )
             return callback(response)
         else:
@@ -77,6 +80,7 @@ def vcr_fetch_impl(cassette, real_fetch_impl):
                         "your current record mode (%r)."
                         % (vcr_request, cassette._path, cassette.record_mode)
                     ),
+                    request_time=self.io_loop.time() - request.start_time,
                 )
                 return callback(response)
 
@@ -93,6 +97,7 @@ def vcr_fetch_impl(cassette, real_fetch_impl):
                     },
                     'headers': headers,
                     'body': {'string': response.body},
+                    'url': response.effective_url,
                 }
                 cassette.append(vcr_request, vcr_response)
                 return callback(response)