v1.7.2

Set request_time on Tornadoo HTTPResponses

README.rst

@@ -608,6 +608,10 @@ new API in version 1.0.x
 
 Changelog
 ---------
+-  1.7.2 [#186] Get effective_url in tornado (thanks @mvschaik), [#187]
+   Set request_time on Response object in tornado (thanks @abhinav).
+-  1.7.1 [#183] Patch ``fetch_impl`` instead of the entire HTTPClient
+   class for Tornado (thanks @abhinav).
 -  1.7.0 [#177] Properly support coroutine/generator decoration. [#178]
    Support distribute (thanks @graingert). [#163] Make compatibility
    between python2 and python3 recorded cassettes more robust (thanks

setup.py

@@ -9,6 +9,7 @@ import pkg_resources
 
 long_description = open('README.rst', 'r').read()
 
+
 class PyTest(TestCommand):
 
     def finalize_options(self):
@@ -23,7 +24,7 @@ class PyTest(TestCommand):
         sys.exit(errno)
 
 
-install_requires=['PyYAML', 'wrapt', 'six>=1.5']
+install_requires = ['PyYAML', 'wrapt', 'six>=1.5']
 
 
 extras_require = {
@@ -47,9 +48,10 @@ except Exception:
         if key.startswith(':'):
             install_requires.extend(value)
 
+
 setup(
     name='vcrpy',
-    version='1.7.0',
+    version='1.7.2',
     description=(
         "Automatically mock your HTTP interactions to simplify and "
         "speed up testing"

tests/integration/test_httplib2.py

@@ -56,6 +56,17 @@ def test_response_headers(scheme, tmpdir):
         resp, _ = httplib2.Http().request(url)
         assert set(headers) == set(resp.items())
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        resp, _ = httplib2.Http().request(url)
+        effective_url = resp['content-location']
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        resp, _ = httplib2.Http().request(url)
+        assert effective_url == resp['content-location']
 
 def test_multiple_requests(scheme, tmpdir):
     '''Ensure that we can cache multiple requests'''

tests/integration/test_requests.py

@@ -44,6 +44,15 @@ def test_body(tmpdir, scheme):
     with vcr.use_cassette(str(tmpdir.join('body.yaml'))):
         assert content == requests.get(url).content
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        effective_url = requests.get(url).url
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        assert effective_url == requests.get(url).url
 
 def test_auth(tmpdir, scheme):
     '''Ensure that we can handle basic auth'''

tests/integration/test_tornado.py

@@ -81,6 +81,17 @@ def test_body(get_client, tmpdir, scheme):
         assert content == (yield get(get_client(), url)).body
         assert 1 == cass.play_count
 
+@pytest.mark.gen_test
+def test_effective_url(get_client, scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))):
+        effective_url = (yield get(get_client(), url)).effective_url
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('url.yaml'))) as cass:
+        assert effective_url == (yield get(get_client(), url)).effective_url
+        assert 1 == cass.play_count
 
 @pytest.mark.gen_test
 def test_auth(get_client, tmpdir, scheme):
@@ -298,3 +309,47 @@ def test_tornado_exception_can_be_caught(get_client):
         yield get(get_client(), 'http://httpbin.org/status/404')
     except http.HTTPError as e:
         assert e.code == 404
+
+
+@pytest.mark.gen_test
+def test_existing_references_get_patched(tmpdir):
+    from tornado.httpclient import AsyncHTTPClient
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))):
+        client = AsyncHTTPClient()
+        yield get(client, 'http://httpbin.org/get')
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))) as cass:
+        yield get(client, 'http://httpbin.org/get')
+        assert cass.play_count == 1
+
+
+@pytest.mark.gen_test
+def test_existing_instances_get_patched(get_client, tmpdir):
+    '''Ensure that existing instances of AsyncHTTPClient get patched upon
+    entering VCR context.'''
+
+    client = get_client()
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))):
+        yield get(client, 'http://httpbin.org/get')
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))) as cass:
+        yield get(client, 'http://httpbin.org/get')
+        assert cass.play_count == 1
+
+
+@pytest.mark.gen_test
+def test_request_time_is_set(get_client, tmpdir):
+    '''Ensures that the request_time on HTTPResponses is set.'''
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))):
+        client = get_client()
+        response = yield get(client, 'http://httpbin.org/get')
+        assert response.request_time is not None
+
+    with vcr.use_cassette(str(tmpdir.join('data.yaml'))) as cass:
+        client = get_client()
+        response = yield get(client, 'http://httpbin.org/get')
+        assert response.request_time is not None
+        assert cass.play_count == 1

tests/integration/test_urllib2.py

@@ -49,6 +49,15 @@ def test_response_headers(scheme, tmpdir):
         open2 = urlopen(url).info().items()
         assert sorted(open1) == sorted(open2)
 
+def test_effective_url(scheme, tmpdir):
+    '''Ensure that the effective_url is captured'''
+    url = scheme + '://httpbin.org/redirect-to?url=/html'
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        effective_url = urlopen(url).geturl()
+        assert effective_url == scheme + '://httpbin.org/html'
+
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))) as cass:
+        assert effective_url == urlopen(url).geturl()
 
 def test_multiple_requests(scheme, tmpdir):
     '''Ensure that we can cache multiple requests'''

vcr/cassette.py

@@ -20,10 +20,18 @@ class CassetteContextDecorator(object):
     """Context manager/decorator that handles installing the cassette and
     removing cassettes.
 
-    This class defers the creation of a new cassette instance until the point at
+    This class defers the creation of a new cassette instance until
-    which it is installed by context manager or decorator. The fact that a new
+    the point at which it is installed by context manager or
-    cassette is used with each application prevents the state of any cassette
+    decorator. The fact that a new cassette is used with each
-    from interfering with another.
+    application prevents the state of any cassette from interfering
+    with another.
+
+    Instances of this class are NOT reentrant as context managers.
+    However, functions that are decorated by
+    ``CassetteContextDecorator`` instances ARE reentrant. See the
+    implementation of ``__call__`` on this class for more details.
+    There is also a guard against attempts to reenter instances of
+    this class as a context manager in ``__exit__``.
     """
 
     _non_cassette_arguments = ('path_transformer', 'func_path_generator')
@@ -41,9 +49,14 @@ class CassetteContextDecorator(object):
         with contextlib.ExitStack() as exit_stack:
             for patcher in CassettePatcherBuilder(cassette).build():
                 exit_stack.enter_context(patcher)
-            log.debug('Entered context for cassette at {0}.'.format(cassette._path))
+            log_format = '{action} context for cassette at {path}.'
+            log.debug(log_format.format(
+                action="Entering", path=cassette._path
+            ))
             yield cassette
-            log.debug('Exiting context for cassette at {0}.'.format(cassette._path))
+            log.debug(log_format.format(
+                action="Exiting", path=cassette._path
+            ))
             # TODO(@IvanMalison): Hmmm. it kind of feels like this should be
             # somewhere else.
             cassette._save()
@@ -78,7 +91,9 @@ class CassetteContextDecorator(object):
         # functions are reentrant. This is required for thread
         # safety and the correct operation of recursive functions.
         args_getter = self._build_args_getter_for_decorator(function)
-        return type(self)(self.cls, args_getter)._execute_function(function, args, kwargs)
+        return type(self)(self.cls, args_getter)._execute_function(
+            function, args, kwargs
+        )
 
     def _execute_function(self, function, args, kwargs):
         if inspect.isgeneratorfunction(function):

vcr/patch.py

@@ -54,13 +54,12 @@ else:
 
 # Try to save the original types for Tornado
 try:
-    import tornado.httpclient
     import tornado.simple_httpclient
 except ImportError:  # pragma: no cover
     pass
 else:
-    _AsyncHTTPClient = tornado.httpclient.AsyncHTTPClient
+    _SimpleAsyncHTTPClient_fetch_impl = \
-    _SimpleAsyncHTTPClient = tornado.simple_httpclient.SimpleAsyncHTTPClient
+            tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch_impl
 
 
 try:
@@ -68,7 +67,8 @@ try:
 except ImportError:  # pragma: no cover
     pass
 else:
-    _CurlAsyncHTTPClient = tornado.curl_httpclient.CurlAsyncHTTPClient
+    _CurlAsyncHTTPClient_fetch_impl = \
+            tornado.curl_httpclient.CurlAsyncHTTPClient.fetch_impl
 
 
 class CassettePatcherBuilder(object):
@@ -228,23 +228,27 @@ class CassettePatcherBuilder(object):
     @_build_patchers_from_mock_triples_decorator
     def _tornado(self):
         try:
-            import tornado.httpclient as http
             import tornado.simple_httpclient as simple
         except ImportError:  # pragma: no cover
             pass
         else:
-            from .stubs.tornado_stubs import VCRAsyncHTTPClient
+            from .stubs.tornado_stubs import vcr_fetch_impl
-            from .stubs.tornado_stubs import VCRSimpleAsyncHTTPClient
 
-            yield http, 'AsyncHTTPClient', VCRAsyncHTTPClient
+            new_fetch_impl = vcr_fetch_impl(
-            yield simple, 'SimpleAsyncHTTPClient', VCRSimpleAsyncHTTPClient
+                self._cassette, _SimpleAsyncHTTPClient_fetch_impl
+            )
+            yield simple.SimpleAsyncHTTPClient, 'fetch_impl', new_fetch_impl
         try:
             import tornado.curl_httpclient as curl
         except ImportError:  # pragma: no cover
             pass
         else:
-            from .stubs.tornado_stubs import VCRCurlAsyncHTTPClient
+            from .stubs.tornado_stubs import vcr_fetch_impl
-            yield curl, 'CurlAsyncHTTPClient', VCRCurlAsyncHTTPClient
+
+            new_fetch_impl = vcr_fetch_impl(
+                self._cassette, _CurlAsyncHTTPClient_fetch_impl
+            )
+            yield curl.CurlAsyncHTTPClient, 'fetch_impl', new_fetch_impl
 
     def _urllib3_patchers(self, cpool, stubs):
         http_connection_remover = ConnectionRemover(
@@ -362,19 +366,25 @@ def reset_patchers():
                                 _CertValidatingHTTPSConnection)
 
     try:
-        import tornado.httpclient as http
         import tornado.simple_httpclient as simple
     except ImportError:  # pragma: no cover
         pass
     else:
-        yield mock.patch.object(http, 'AsyncHTTPClient', _AsyncHTTPClient)
+        yield mock.patch.object(
-        yield mock.patch.object(simple, 'SimpleAsyncHTTPClient', _SimpleAsyncHTTPClient)
+            simple.SimpleAsyncHTTPClient,
+            'fetch_impl',
+            _SimpleAsyncHTTPClient_fetch_impl,
+        )
     try:
         import tornado.curl_httpclient as curl
     except ImportError:  # pragma: no cover
         pass
     else:
-        yield mock.patch.object(curl, 'CurlAsyncHTTPClient', _CurlAsyncHTTPClient)
+        yield mock.patch.object(
+            curl.CurlAsyncHTTPClient,
+            'fetch_impl', 
+            _CurlAsyncHTTPClient_fetch_impl,
+        )
 
 
 @contextlib.contextmanager

vcr/stubs/tornado_stubs.py

@@ -1,48 +1,20 @@
 '''Stubs for tornado HTTP clients'''
 from __future__ import absolute_import
 
+import functools
 from six import BytesIO
 
 from tornado import httputil
-from tornado.httpclient import AsyncHTTPClient
 from tornado.httpclient import HTTPResponse
-from tornado.simple_httpclient import SimpleAsyncHTTPClient
 
 from vcr.errors import CannotOverwriteExistingCassetteException
 from vcr.request import Request
 
 
-class _VCRAsyncClient(object):
+def vcr_fetch_impl(cassette, real_fetch_impl):
-    cassette = None
 
-    def __new__(cls, *args, **kwargs):
+    @functools.wraps(real_fetch_impl)
-        from vcr.patch import force_reset
+    def new_fetch_impl(self, request, callback):
-        with force_reset():
-            return super(_VCRAsyncClient, cls).__new__(cls, *args, **kwargs)
-
-    def initialize(self, *args, **kwargs):
-        from vcr.patch import force_reset
-        with force_reset():
-            self.real_client = self._baseclass(*args, **kwargs)
-
-    @property
-    def io_loop(self):
-        return self.real_client.io_loop
-
-    @property
-    def _closed(self):
-        return self.real_client._closed
-
-    @property
-    def defaults(self):
-        return self.real_client.defaults
-
-    def close(self):
-        from vcr.patch import force_reset
-        with force_reset():
-            self.real_client.close()
-
-    def fetch_impl(self, request, callback):
         headers = dict(request.headers)
         if request.user_agent:
             headers.setdefault('User-Agent', request.user_agent)
@@ -64,6 +36,7 @@ class _VCRAsyncClient(object):
                     "that is not yet supported by VCR.py. Please make the "
                     "request outside a VCR.py context." % repr(request)
                 ),
+                request_time=self.io_loop.time() - request.start_time,
             )
             return callback(response)
 
@@ -74,8 +47,8 @@ class _VCRAsyncClient(object):
             headers,
         )
 
-        if self.cassette.can_play_response_for(vcr_request):
+        if cassette.can_play_response_for(vcr_request):
-            vcr_response = self.cassette.play_response(vcr_request)
+            vcr_response = cassette.play_response(vcr_request)
             headers = httputil.HTTPHeaders()
 
             recorded_headers = vcr_response['headers']
@@ -90,10 +63,12 @@ class _VCRAsyncClient(object):
                 reason=vcr_response['status']['message'],
                 headers=headers,
                 buffer=BytesIO(vcr_response['body']['string']),
+                effective_url=vcr_response.get('url'),
+                request_time=self.io_loop.time() - request.start_time,
             )
             return callback(response)
         else:
-            if self.cassette.write_protected and self.cassette.filter_request(
+            if cassette.write_protected and cassette.filter_request(
                 vcr_request
             ):
                 response = HTTPResponse(
@@ -103,9 +78,9 @@ class _VCRAsyncClient(object):
                         "No match for the request (%r) was found. "
                         "Can't overwrite existing cassette (%r) in "
                         "your current record mode (%r)."
-                        % (vcr_request, self.cassette._path,
+                        % (vcr_request, cassette._path, cassette.record_mode)
-                           self.cassette.record_mode)
                     ),
+                    request_time=self.io_loop.time() - request.start_time,
                 )
                 return callback(response)
 
@@ -122,27 +97,11 @@ class _VCRAsyncClient(object):
                     },
                     'headers': headers,
                     'body': {'string': response.body},
+                    'url': response.effective_url,
                 }
-                self.cassette.append(vcr_request, vcr_response)
+                cassette.append(vcr_request, vcr_response)
                 return callback(response)
 
-            from vcr.patch import force_reset
+            real_fetch_impl(self, request, new_callback)
-            with force_reset():
-                self.real_client.fetch_impl(request, new_callback)
 
-
+    return new_fetch_impl
-class VCRAsyncHTTPClient(_VCRAsyncClient, AsyncHTTPClient):
-    _baseclass = AsyncHTTPClient
-
-
-class VCRSimpleAsyncHTTPClient(_VCRAsyncClient, SimpleAsyncHTTPClient):
-    _baseclass = SimpleAsyncHTTPClient
-
-
-try:
-    from tornado.curl_httpclient import CurlAsyncHTTPClient
-except ImportError:  # pragma: no cover
-    VCRCurlAsyncHTTPClient = None
-else:
-    class VCRCurlAsyncHTTPClient(_VCRAsyncClient, CurlAsyncHTTPClient):
-        _baseclass = CurlAsyncHTTPClient