From 43f4eb81569323c1be859c1f32da25f1f601fc28 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Thu, 22 Jun 2017 16:49:59 +0100 Subject: [PATCH 01/64] Fix host and port for proxy connections --- vcr/stubs/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 5ab819d..52b57f4 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -18,7 +18,7 @@ log = logging.getLogger(__name__) class VCRFakeSocket(object): """ A socket that doesn't do anything! - Used when playing back casssettes, when there + Used when playing back cassettes, when there is no actual open socket. """ @@ -36,6 +36,16 @@ class VCRFakeSocket(object): """ return 0 # wonder how bad this is.... + def __nonzero__(self): + """This is hacky too. + + urllib3 checks if sock is truthy before calling + set_tunnel (urllib3/connectionpool.py#L592). + If it is true, it never sets the tunnel and this + breaks proxy requests. + """ + return False + def parse_headers(header_list): """ @@ -130,7 +140,7 @@ class VCRConnection(object): """ Returns empty string for the default port and ':port' otherwise """ - port = self.real_connection.port + port = self._tunnel_port or self.real_connection.port default_port = {'https': 443, 'http': 80}[self._protocol] return ':{}'.format(port) if port != default_port else '' @@ -138,7 +148,7 @@ class VCRConnection(object): """Returns request absolute URI""" uri = "{}://{}{}{}".format( self._protocol, - self.real_connection.host, + self._tunnel_host or self.real_connection.host, self._port_postfix(), url, ) @@ -148,7 +158,7 @@ class VCRConnection(object): """Returns request selector url from absolute URI""" prefix = "{}://{}{}".format( self._protocol, - self.real_connection.host, + self._tunnel_host or self.real_connection.host, self._port_postfix(), ) return uri.replace(prefix, '', 1) @@ -313,6 +323,9 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) + self._tunnel_host = None + self._tunnel_port = None + def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 236dc1f4f299d85af8ab38e19c16aac828556f2b Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 12:25:01 +0100 Subject: [PATCH 02/64] Add test for proxies --- tests/integration/test_proxy.py | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/integration/test_proxy.py diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py new file mode 100644 index 0000000..3212b24 --- /dev/null +++ b/tests/integration/test_proxy.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +'''Test using a proxy.''' + +# External imports +import multiprocessing +import SocketServer +import SimpleHTTPServer +import pytest +requests = pytest.importorskip("requests") + +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr + + +class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): + ''' + Simple proxy server. + + (from: http://effbot.org/librarybook/simplehttpserver.htm). + ''' + def do_GET(self): + self.copyfile(urlopen(self.path), self.wfile) + + +@pytest.yield_fixture(scope='session') +def proxy_server(httpbin): + httpd = SocketServer.ForkingTCPServer(('', 0), Proxy) + proxy_process = multiprocessing.Process( + target=httpd.serve_forever, + ) + proxy_process.start() + yield 'http://{}:{}'.format(*httpd.server_address) + proxy_process.terminate() + + +def test_use_proxy(tmpdir, httpbin, proxy_server): + '''Ensure that it works with a proxy.''' + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cass: + requests.get(httpbin.url, proxies={'http': proxy_server}) + requests.get(httpbin.url, proxies={'http': proxy_server}) From fc95e34bd4d260cf8bc4199c91cf3f285bf346ab Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 13:13:02 +0100 Subject: [PATCH 03/64] Determine proxy based on path --- vcr/stubs/__init__.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 52b57f4..6ebe2e8 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -36,16 +36,6 @@ class VCRFakeSocket(object): """ return 0 # wonder how bad this is.... - def __nonzero__(self): - """This is hacky too. - - urllib3 checks if sock is truthy before calling - set_tunnel (urllib3/connectionpool.py#L592). - If it is true, it never sets the tunnel and this - breaks proxy requests. - """ - return False - def parse_headers(header_list): """ @@ -140,15 +130,18 @@ class VCRConnection(object): """ Returns empty string for the default port and ':port' otherwise """ - port = self._tunnel_port or self.real_connection.port + port = self.real_connection.port default_port = {'https': 443, 'http': 80}[self._protocol] return ':{}'.format(port) if port != default_port else '' def _uri(self, url): """Returns request absolute URI""" - uri = "{}://{}{}{}".format( + if not url.startswith('/'): + # Then this must be a proxy request. + return url + uri = "{0}://{1}{2}{3}".format( self._protocol, - self._tunnel_host or self.real_connection.host, + self.real_connection.host, self._port_postfix(), url, ) @@ -158,7 +151,7 @@ class VCRConnection(object): """Returns request selector url from absolute URI""" prefix = "{}://{}{}".format( self._protocol, - self._tunnel_host or self.real_connection.host, + self.real_connection.host, self._port_postfix(), ) return uri.replace(prefix, '', 1) @@ -323,9 +316,6 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) - self._tunnel_host = None - self._tunnel_port = None - def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 365a98bf665e5e9f4be025f47a10d8b3b1451343 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 15:16:23 +0100 Subject: [PATCH 04/64] Fix failing tests --- tests/integration/test_proxy.py | 2 +- vcr/stubs/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 3212b24..ce67e60 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -37,6 +37,6 @@ def proxy_server(httpbin): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' - with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): requests.get(httpbin.url, proxies={'http': proxy_server}) requests.get(httpbin.url, proxies={'http': proxy_server}) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 6ebe2e8..763204e 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -136,7 +136,7 @@ class VCRConnection(object): def _uri(self, url): """Returns request absolute URI""" - if not url.startswith('/'): + if url and not url.startswith('/'): # Then this must be a proxy request. return url uri = "{0}://{1}{2}{3}".format( From bed9e520a371a99132e05511f110a141d22d2a7f Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 15:29:36 +0100 Subject: [PATCH 05/64] Fix `socketserver` for Python 3 --- tests/integration/test_proxy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index ce67e60..ab104b1 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -3,11 +3,10 @@ # External imports import multiprocessing -import SocketServer -import SimpleHTTPServer import pytest requests = pytest.importorskip("requests") +from six.moves import socketserver, SimpleHTTPServer from six.moves.urllib.request import urlopen # Internal imports @@ -26,7 +25,7 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): @pytest.yield_fixture(scope='session') def proxy_server(httpbin): - httpd = SocketServer.ForkingTCPServer(('', 0), Proxy) + httpd = socketserver.ForkingTCPServer(('', 0), Proxy) proxy_process = multiprocessing.Process( target=httpd.serve_forever, ) From eb4774a7d2252789f34801d9026e3d3cabd3be03 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 16:54:29 +0100 Subject: [PATCH 06/64] Only have a sock attribute after connecting --- vcr/stubs/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 763204e..a23a731 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -171,6 +171,8 @@ class VCRConnection(object): # allows me to compare the entire length of the response to see if it # exists in the cassette. + self._sock = VCRFakeSocket() + def putrequest(self, method, url, *args, **kwargs): """ httplib gives you more than one way to do it. This is a way @@ -294,11 +296,13 @@ class VCRConnection(object): with force_reset(): return self.real_connection.connect(*args, **kwargs) + self._sock = VCRFakeSocket() + @property def sock(self): if self.real_connection.sock: return self.real_connection.sock - return VCRFakeSocket() + return self._sock @sock.setter def sock(self, value): @@ -316,6 +320,8 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) + self._sock = None + def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 06dc2190d64e312b3b8285e69a0d50342bc55b46 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Wed, 18 Oct 2017 12:44:52 +0100 Subject: [PATCH 07/64] Fix format string for Python 2.6 --- tests/integration/test_proxy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index ab104b1..d3c8df1 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -4,7 +4,6 @@ # External imports import multiprocessing import pytest -requests = pytest.importorskip("requests") from six.moves import socketserver, SimpleHTTPServer from six.moves.urllib.request import urlopen @@ -12,6 +11,9 @@ from six.moves.urllib.request import urlopen # Internal imports import vcr +# Conditional imports +requests = pytest.importorskip("requests") + class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' @@ -24,13 +26,13 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): @pytest.yield_fixture(scope='session') -def proxy_server(httpbin): - httpd = socketserver.ForkingTCPServer(('', 0), Proxy) +def proxy_server(): + httpd = socketserver.ThreadingTCPServer(('', 0), Proxy) proxy_process = multiprocessing.Process( target=httpd.serve_forever, ) proxy_process.start() - yield 'http://{}:{}'.format(*httpd.server_address) + yield 'http://{0}:{1}'.format(*httpd.server_address) proxy_process.terminate() From 4b6b5effc744583eb0227d14d0c5e324c50cf074 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Fri, 3 Nov 2017 11:07:23 +0000 Subject: [PATCH 08/64] Add headers in proxy server response --- tests/integration/test_proxy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index d3c8df1..a1fc003 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -19,10 +19,15 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' Simple proxy server. - (from: http://effbot.org/librarybook/simplehttpserver.htm). + (Inspired by: http://effbot.org/librarybook/simplehttpserver.htm). ''' def do_GET(self): - self.copyfile(urlopen(self.path), self.wfile) + upstream_response = urlopen(self.path) + self.send_response(upstream_response.status, upstream_response.msg) + for header in upstream_response.headers.items(): + self.send_header(*header) + self.end_headers() + self.copyfile(upstream_response, self.wfile) @pytest.yield_fixture(scope='session') From ff7dd06f4730041db3a9b864d6721a59dffaa94a Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Fri, 3 Nov 2017 11:39:13 +0000 Subject: [PATCH 09/64] fix proxy for Python 2 --- tests/integration/test_proxy.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index a1fc003..5dfd285 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -23,8 +23,15 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' def do_GET(self): upstream_response = urlopen(self.path) - self.send_response(upstream_response.status, upstream_response.msg) - for header in upstream_response.headers.items(): + try: + status = upstream_response.status + headers = upstream_response.headers.items() + except AttributeError: + # In Python 2 the response is an addinfourl instance. + status = upstream_response.code + headers = upstream_response.info().items() + self.send_response(status, upstream_response.msg) + for header in headers: self.send_header(*header) self.end_headers() self.copyfile(upstream_response, self.wfile) From bb8d39dd20683a6318a682df815ed10363bd97f6 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 18 Jul 2018 22:51:58 +0100 Subject: [PATCH 10/64] Update docs' lists of supported HTTP clients I noticed these were out of sync, especially weirdly one mentioning boto and the other mentioning boto3. I figure the list in the README is redundant when the Installation docs section lists the supported libraries, so I've removed it. I also: * alphabetically sorted the list * Highlighted the library names as code * Added both `boto` and `boto3` to the list since there is support for both * Removed the comment about Tornado's AsyncHTTPClient since that's an implementation detail plus the patch happens on a couple different classes * Removed the note about `http.client` being part of Python 3, because everyone is Python 3 these days anyway :) --- README.rst | 13 ------------- docs/installation.rst | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 8c45df3..3386713 100644 --- a/README.rst +++ b/README.rst @@ -41,19 +41,6 @@ VCR.py will detect the absence of a cassette file and once again record all HTTP interactions, which will update them to correspond to the new API. -Support -------- - -VCR.py works great with the following HTTP clients: - -- requests -- aiohttp -- urllib3 -- tornado -- urllib2 -- boto3 - - License ======= diff --git a/docs/installation.rst b/docs/installation.rst index cc99e34..adaf070 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -12,15 +12,17 @@ Compatibility VCR.py supports Python 2.7 and 3.4+, and `pypy `__. -The following http libraries are supported: +The following HTTP libraries are supported: -- urllib2 -- urllib3 -- http.client (python3) -- requests (both 1.x and 2.x versions) -- httplib2 -- boto -- Tornado's AsyncHTTPClient +- ``aiohttp`` +- ``boto`` +- ``boto3`` +- ``http.client`` +- ``httplib2`` +- ``requests`` (both 1.x and 2.x versions) +- ``tornado`` +- ``urllib2`` +- ``urllib3`` Speed ----- From e559be758ad6d22fed9ef662aac848e527329e89 Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Thu, 28 Jun 2018 09:12:26 -0700 Subject: [PATCH 11/64] Fix aiohttp patch to work with aiohttp >= 3.3 Aiohttp expects an awaitable instance to be returned from `ClientSession._request` though `asyncio.coroutine` decorated function do not implement `__await__`. By changing the syntax and dropping Python 3.4 support we fix this issue. --- tests/integration/test_aiohttp.py | 12 ++++++++++++ tox.ini | 1 + vcr/stubs/aiohttp_stubs/__init__.py | 7 +++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index 6cc28c3..4951e69 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -154,3 +154,15 @@ def test_params_on_url(tmpdir, scheme): assert request.url == url assert cassette_response_json == response_json assert cassette.play_count == 1 + + +async def test_aiohttp_client_does_not_break_with_patching_request(aiohttp_client, tmpdir): + async def hello(request): + return aiohttp.web.Response(text='Hello, world') + + app = aiohttp.web.Application() + app.router.add_get('/', hello) + client = await aiohttp_client(app) + + with vcr.use_cassette(str(tmpdir.join('get.yaml'))): + await client.get('/') diff --git a/tox.ini b/tox.ini index b72d43b..3eeb686 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,7 @@ deps = boto3: boto3 aiohttp: aiohttp aiohttp: pytest-asyncio + aiohttp: pytest-aiohttp [flake8] max_line_length = 110 diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index f57cb63..78bb658 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -45,8 +45,7 @@ class MockClientResponse(ClientResponse): def vcr_request(cassette, real_request): @functools.wraps(real_request) - @asyncio.coroutine - def new_request(self, method, url, **kwargs): + async def new_request(self, method, url, **kwargs): headers = kwargs.get('headers') headers = self._prepare_headers(headers) data = kwargs.get('data') @@ -82,7 +81,7 @@ def vcr_request(cassette, real_request): response.close() return response - response = yield from real_request(self, method, url, **kwargs) # NOQA: E999 + response = await real_request(self, method, url, **kwargs) # NOQA: E999 vcr_response = { 'status': { @@ -90,7 +89,7 @@ def vcr_request(cassette, real_request): 'message': response.reason, }, 'headers': dict(response.headers), - 'body': {'string': (yield from response.read())}, # NOQA: E999 + 'body': {'string': (await response.read())}, # NOQA: E999 'url': response.url, } cassette.append(vcr_request, vcr_response) From a9e75a545e04a63bd8ee4749d88bbcc9656831ee Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 31 Jul 2018 09:54:29 +0100 Subject: [PATCH 12/64] Update docs on before_record_* callbacks Make them a bit more consistent and obvious that returning `None` ignores the request/response. --- docs/advanced.rst | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 5e6822d..198a3d2 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -221,24 +221,25 @@ Custom Request filtering ~~~~~~~~~~~~~~~~~~~~~~~~ If none of these covers your request filtering needs, you can register a -callback that will manipulate the HTTP request before adding it to the -cassette. Use the ``before_record_request`` configuration option to so this. -Here is an example that will never record requests to the /login -endpoint. +callback with the ``before_record_request`` configuration option to +manipulate the HTTP request before adding it to the cassette, or return +``None`` to ignore it entirely. Here is an example that will never record +requests to the ``'/login'`` path: .. code:: python def before_record_cb(request): - if request.path != '/login': - return request + if request.path == '/login': + return None + return request my_vcr = vcr.VCR( - before_record_request = before_record_cb, + before_record_request=before_record_cb, ) with my_vcr.use_cassette('test.yml'): # your http code here -You can also mutate the response using this callback. For example, you +You can also mutate the request using this callback. For example, you could remove all query parameters from any requests to the ``'/login'`` path. @@ -246,7 +247,7 @@ path. def scrub_login_request(request): if request.path == '/login': - request.uri, _ = urllib.splitquery(response.uri) + request.uri, _ = urllib.splitquery(request.uri) return request my_vcr = vcr.VCR( @@ -258,9 +259,12 @@ path. Custom Response Filtering ~~~~~~~~~~~~~~~~~~~~~~~~~ -VCR.py also suports response filtering with the -``before_record_response`` keyword argument. It's usage is similar to -that of ``before_record``: +You can also do response filtering with the +``before_record_response`` configuration option. Its usage is +similar to the above ``before_record_request`` - you can +mutate the response, or return ``None`` to avoid recording +the request and response altogether. For example to hide +sensitive data from the request body: .. code:: python @@ -302,8 +306,8 @@ in a few ways: or 0.0.0.0. - Set the ``ignore_hosts`` configuration option to a list of hosts to ignore -- Add a ``before_record`` callback that returns None for requests you - want to ignore +- Add a ``before_record_request`` or ``before_record_response`` callback + that returns ``None`` for requests you want to ignore (see above). Requests that are ignored by VCR will not be saved in a cassette, nor played back from a cassette. VCR will completely ignore those requests From 7417978e368889184f32d76f83dc3c8e2862143f Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 31 Jul 2018 10:38:21 +0100 Subject: [PATCH 13/64] tornado.httpclient --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index adaf070..afec93b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -20,7 +20,7 @@ The following HTTP libraries are supported: - ``http.client`` - ``httplib2`` - ``requests`` (both 1.x and 2.x versions) -- ``tornado`` +- ``tornado.httpclient`` - ``urllib2`` - ``urllib3`` From 76076e5ccb600e62e47f57d2a23815dc85949ff1 Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Wed, 5 Sep 2018 20:05:27 +0300 Subject: [PATCH 14/64] Fix PyPI badge URLs in README.rst `vcrpy-unittest` -> `vcrpy` in PyPI URLs. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8c45df3..cfa0b38 100644 --- a/README.rst +++ b/README.rst @@ -61,9 +61,9 @@ This library uses the MIT license. See `LICENSE.txt `__ for more details .. |PyPI| image:: https://img.shields.io/pypi/v/vcrpy.svg - :target: https://pypi.python.org/pypi/vcrpy-unittest -.. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy-unittest.svg - :target: https://pypi.python.org/pypi/vcrpy-unittest + :target: https://pypi.python.org/pypi/vcrpy +.. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy.svg + :target: https://pypi.python.org/pypi/vcrpy .. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.png?branch=master :target: http://travis-ci.org/kevin1024/vcrpy .. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.png?label=ready&title=waffle From 5ddcaa4870d0ae1e67b5d2bb4798a6afbc6dd294 Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Wed, 5 Sep 2018 20:26:01 +0300 Subject: [PATCH 15/64] Replace PNG badges with SVG ones --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cfa0b38..ca3bdf4 100644 --- a/README.rst +++ b/README.rst @@ -64,9 +64,9 @@ more details :target: https://pypi.python.org/pypi/vcrpy .. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy.svg :target: https://pypi.python.org/pypi/vcrpy -.. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.png?branch=master +.. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.svg?branch=master :target: http://travis-ci.org/kevin1024/vcrpy -.. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.png?label=ready&title=waffle +.. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.svg?label=ready&title=waffle :target: https://waffle.io/kevin1024/vcrpy .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg :alt: Join the chat at https://gitter.im/kevin1024/vcrpy From 957db22d5c50b560b923184da04e06b63afb5f60 Mon Sep 17 00:00:00 2001 From: Danilo Shiga Date: Tue, 18 Sep 2018 10:11:33 -0300 Subject: [PATCH 16/64] Improve test_use_proxy with cassette headers and play_count assertion --- tests/integration/test_proxy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 5dfd285..4dfcb92 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -51,5 +51,10 @@ def proxy_server(): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): - requests.get(httpbin.url, proxies={'http': proxy_server}) - requests.get(httpbin.url, proxies={'http': proxy_server}) + response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette: + cassette_response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + + assert cassette_response.headers == response.headers + assert cassette.play_count == 1 From e9d00a5e2aa304815036ff14008ade2fec06a514 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Tue, 18 Sep 2018 11:13:39 -0300 Subject: [PATCH 17/64] Fix test_use_proxy cassette response type --- tests/integration/test_proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 4dfcb92..9b3c4ad 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -51,10 +51,10 @@ def proxy_server(): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): - response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + response = requests.get(httpbin.url, proxies={'http': proxy_server}) with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette: - cassette_response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + cassette_response = requests.get(httpbin.url, proxies={'http': proxy_server}) assert cassette_response.headers == response.headers assert cassette.play_count == 1 From f7c051cde68380dac5e455046b775cc0594e9eff Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Tue, 18 Sep 2018 13:59:40 -0300 Subject: [PATCH 18/64] Drop support to asyncio.coroutine (py34 async/await syntax) --- tests/integration/aiohttp_utils.py | 23 ++++++++++++++++------- tests/integration/test_aiohttp.py | 29 ++++++++++++++++++++--------- vcr/_handle_coroutine.py | 8 ++------ vcr/stubs/aiohttp_stubs/__init__.py | 13 ++++--------- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/tests/integration/aiohttp_utils.py b/tests/integration/aiohttp_utils.py index b0b2dd6..6f6f15e 100644 --- a/tests/integration/aiohttp_utils.py +++ b/tests/integration/aiohttp_utils.py @@ -2,23 +2,32 @@ import asyncio import aiohttp +from aiohttp.test_utils import TestClient -@asyncio.coroutine -def aiohttp_request(loop, method, url, output='text', encoding='utf-8', content_type=None, **kwargs): +async def aiohttp_request(loop, method, url, output='text', encoding='utf-8', content_type=None, **kwargs): session = aiohttp.ClientSession(loop=loop) response_ctx = session.request(method, url, **kwargs) - response = yield from response_ctx.__aenter__() + response = await response_ctx.__aenter__() if output == 'text': - content = yield from response.text() + content = await response.text() elif output == 'json': content_type = content_type or 'application/json' - content = yield from response.json(encoding=encoding, content_type=content_type) + content = await response.json(encoding=encoding, content_type=content_type) elif output == 'raw': - content = yield from response.read() + content = await response.read() response_ctx._resp.close() - yield from session.close() + await session.close() return response, content + + +def aiohttp_app(): + async def hello(request): + return aiohttp.web.Response(text='hello') + + app = aiohttp.web.Application() + app.router.add_get('/', hello) + return app diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index 4951e69..74aaccc 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -5,7 +5,7 @@ asyncio = pytest.importorskip("asyncio") aiohttp = pytest.importorskip("aiohttp") import vcr # noqa: E402 -from .aiohttp_utils import aiohttp_request # noqa: E402 +from .aiohttp_utils import aiohttp_app, aiohttp_request # noqa: E402 def run_in_loop(fn): @@ -156,13 +156,24 @@ def test_params_on_url(tmpdir, scheme): assert cassette.play_count == 1 -async def test_aiohttp_client_does_not_break_with_patching_request(aiohttp_client, tmpdir): - async def hello(request): - return aiohttp.web.Response(text='Hello, world') - - app = aiohttp.web.Application() - app.router.add_get('/', hello) - client = await aiohttp_client(app) +def test_aiohttp_test_client(aiohttp_client, tmpdir): + loop = asyncio.get_event_loop() + app = aiohttp_app() + url = '/' + client = loop.run_until_complete(aiohttp_client(app)) with vcr.use_cassette(str(tmpdir.join('get.yaml'))): - await client.get('/') + response = loop.run_until_complete(client.get(url)) + + assert response.status == 200 + response_text = loop.run_until_complete(response.text()) + assert response_text == 'hello' + + with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette: + response = loop.run_until_complete(client.get(url)) + + request = cassette.requests[0] + assert request.url == str(client.make_url(url)) + response_text = loop.run_until_complete(response.text()) + assert response_text == 'hello' + assert cassette.play_count == 1 diff --git a/vcr/_handle_coroutine.py b/vcr/_handle_coroutine.py index 0b20be6..610305f 100644 --- a/vcr/_handle_coroutine.py +++ b/vcr/_handle_coroutine.py @@ -1,7 +1,3 @@ -import asyncio - - -@asyncio.coroutine -def handle_coroutine(vcr, fn): +async def handle_coroutine(vcr, fn): # noqa: E999 with vcr as cassette: - return (yield from fn(cassette)) # noqa: E999 + return (await fn(cassette)) # noqa: E999 diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index 78bb658..38f337e 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -25,21 +25,16 @@ class MockClientResponse(ClientResponse): session=None, ) - # TODO: get encoding from header - @asyncio.coroutine - def json(self, *, encoding='utf-8', loads=json.loads, **kwargs): # NOQA: E999 + async def json(self, *, encoding='utf-8', loads=json.loads, **kwargs): # NOQA: E999 return loads(self._body.decode(encoding)) - @asyncio.coroutine - def text(self, encoding='utf-8'): + async def text(self, encoding='utf-8'): return self._body.decode(encoding) - @asyncio.coroutine - def read(self): + async def read(self): return self._body - @asyncio.coroutine - def release(self): + async def release(self): pass From e93060c81bef0130d0626b348ab753db50e3f8fb Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sat, 21 Jul 2018 17:03:19 +0800 Subject: [PATCH 19/64] Fix compatibility with Python 3.7 --- setup.py | 1 + tests/integration/test_urllib2.py | 5 ++++- vcr/cassette.py | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3935af6..65c0773 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Testing', diff --git a/tests/integration/test_urllib2.py b/tests/integration/test_urllib2.py index 8a633ba..3c0b021 100644 --- a/tests/integration/test_urllib2.py +++ b/tests/integration/test_urllib2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- '''Integration tests with urllib2''' +import ssl from six.moves.urllib.request import urlopen from six.moves.urllib_parse import urlencode import pytest_httpbin.certs @@ -12,7 +13,9 @@ from assertions import assert_cassette_has_one_response def urlopen_with_cafile(*args, **kwargs): - kwargs['cafile'] = pytest_httpbin.certs.where() + context = ssl.create_default_context(cafile=pytest_httpbin.certs.where()) + context.check_hostname = False + kwargs['context'] = context try: return urlopen(*args, **kwargs) except TypeError: diff --git a/vcr/cassette.py b/vcr/cassette.py index d64dec6..3b6f54e 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -136,7 +136,10 @@ class CassetteContextDecorator(object): except Exception: to_yield = coroutine.throw(*sys.exc_info()) else: - to_yield = coroutine.send(to_send) + try: + to_yield = coroutine.send(to_send) + except StopIteration: + break def _handle_function(self, fn): with self as cassette: From b38915a89aef77f7169fdf3aa949072050029648 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Tue, 18 Sep 2018 18:42:22 -0300 Subject: [PATCH 20/64] Fix httplib2 compatibility with py37 --- tests/integration/test_httplib2.py | 11 ++++++++--- vcr/stubs/httplib2_stubs.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_httplib2.py b/tests/integration/test_httplib2.py index 05a25d3..4eda032 100644 --- a/tests/integration/test_httplib2.py +++ b/tests/integration/test_httplib2.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- '''Integration tests with httplib2''' -# External imports +import sys + from six.moves.urllib_parse import urlencode import pytest import pytest_httpbin.certs -# Internal imports import vcr from assertions import assert_cassette_has_one_response @@ -19,7 +19,12 @@ def http(): Returns an httplib2 HTTP instance with the certificate replaced by the httpbin one. """ - return httplib2.Http(ca_certs=pytest_httpbin.certs.where()) + kwargs = { + 'ca_certs': pytest_httpbin.certs.where() + } + if sys.version_info[:2] == (3, 7): + kwargs['disable_ssl_certificate_validation'] = True + return httplib2.Http(**kwargs) def test_response_code(tmpdir, httpbin_both): diff --git a/vcr/stubs/httplib2_stubs.py b/vcr/stubs/httplib2_stubs.py index 4f31537..3d83406 100644 --- a/vcr/stubs/httplib2_stubs.py +++ b/vcr/stubs/httplib2_stubs.py @@ -40,6 +40,7 @@ class VCRHTTPSConnectionWithTimeout(VCRHTTPSConnection, 'timeout', 'source_address', 'ca_certs', + 'disable_ssl_certificate_validation', } unknown_keys = set(kwargs.keys()) - safe_keys safe_kwargs = kwargs.copy() From 9a9cdb3a950cb6a5197eab1d6c10d9072b4cfccf Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Tue, 18 Sep 2018 18:47:32 -0300 Subject: [PATCH 21/64] add py37 on CI build --- .travis.yml | 25 +++++++++++++++++++++++++ tox.ini | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 169ee5a..6759546 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,31 @@ env: - TOX_SUFFIX="tornado4" - TOX_SUFFIX="aiohttp" matrix: + include: + - env: TOX_SUFFIX="flakes" + python: 3.7 + dist: xenial + sudo: true + - env: TOX_SUFFIX="requests27" + python: 3.7 + dist: xenial + sudo: true + - env: TOX_SUFFIX="httplib2" + python: 3.7 + dist: xenial + sudo: true + - env: TOX_SUFFIX="urllib3121" + python: 3.7 + dist: xenial + sudo: true + - env: TOX_SUFFIX="tornado4" + python: 3.7 + dist: xenial + sudo: true + - env: TOX_SUFFIX="aiohttp" + python: 3.7 + dist: xenial + sudo: true allow_failures: - env: TOX_SUFFIX="boto3" - env: TOX_SUFFIX="aiohttp" diff --git a/tox.ini b/tox.ini index 3eeb686..865cd62 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} +envlist = {py27,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} [testenv:flakes] skipsdist = True From 0cf11d4525a706fd807ffb9f5d35a8c8f7a9ef53 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Wed, 19 Sep 2018 11:46:08 -0300 Subject: [PATCH 22/64] Bump version to 2.0.0 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d3b393..a00dcc6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,12 @@ Changelog --------- +- 2.0.0 - Support python 3.7 (fix httplib2 and urllib2, thanks @felixonmars) + [#356] Fixes `before_record_response` so the original response isn't changed (thanks @kgraves) + Fix requests stub when using proxy (thanks @samuelfekete @daneoshiga) + (only for aiohttp stub) Drop support to python 3.4 asyncio.coroutine (aiohttp doesn't support python it anymore) + Fix aiohttp stub to work with aiohttp client (thanks @stj) + Fix aiohttp stub to accept content type passed + Improve docs (thanks @adamchainz) - 1.13.0 - Fix support to latest aiohttp version (3.3.2). Fix content-type bug in aiohttp stub. Save URL with query params properly when using aiohttp. - 1.12.0 - Fix support to latest aiohttp version (3.2.1), Adapted setup to PEP508, Support binary responses on aiohttp, Dropped support for EOL python versions (2.6 and 3.3) - 1.11.1 Fix compatibility with newest requests and urllib3 releases diff --git a/setup.py b/setup.py index 65c0773..54c0607 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ if sys.version_info[0] == 2: setup( name='vcrpy', - version='1.13.0', + version='2.0.0', description=( "Automatically mock your HTTP interactions to simplify and " "speed up testing" From 287ea4b06e0354f505267273122b18999b34f0b4 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Sat, 22 Sep 2018 17:29:08 -0300 Subject: [PATCH 23/64] Fix cassette module to work with py34 --- vcr/cassette.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index 3b6f54e..5683ddf 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -16,11 +16,13 @@ from .util import partition_dict try: from asyncio import iscoroutinefunction - from ._handle_coroutine import handle_coroutine except ImportError: def iscoroutinefunction(*args, **kwargs): return False +if sys.version_info[:2] >= (3, 5): + from ._handle_coroutine import handle_coroutine +else: def handle_coroutine(*args, **kwags): raise NotImplementedError('Not implemented on Python 2') From f2a79d3fcca4d3c069c5f83f633977e333cb9433 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Sat, 22 Sep 2018 17:30:21 -0300 Subject: [PATCH 24/64] Add py34 to CI builds --- .travis.yml | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6759546..17a2546 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,8 @@ matrix: - env: TOX_SUFFIX="boto3" - env: TOX_SUFFIX="aiohttp" python: "pypy3.5-5.9.0" + - env: TOX_SUFFIX="aiohttp" + python: 3.4 exclude: # Only run flakes on a single Python 2.x and a single 3.x - env: TOX_SUFFIX="flakes" @@ -59,6 +61,7 @@ matrix: python: pypy python: - 2.7 +- 3.4 - 3.5 - 3.6 - pypy diff --git a/tox.ini b/tox.ini index 865cd62..bd99f42 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} +envlist = {py27,py34,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} [testenv:flakes] skipsdist = True From e42746fa88a400ae87551fc5e6274eb6c71b03c6 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Sun, 23 Sep 2018 15:25:04 -0300 Subject: [PATCH 25/64] Bump version to 2.0.1 --- docs/changelog.rst | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a00dcc6..8ff6e58 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,6 @@ Changelog --------- +- 2.0.1 - Fix bug when using vcrpy with python 3.4 - 2.0.0 - Support python 3.7 (fix httplib2 and urllib2, thanks @felixonmars) [#356] Fixes `before_record_response` so the original response isn't changed (thanks @kgraves) Fix requests stub when using proxy (thanks @samuelfekete @daneoshiga) diff --git a/setup.py b/setup.py index 54c0607..ae821ac 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ if sys.version_info[0] == 2: setup( name='vcrpy', - version='2.0.0', + version='2.0.1', description=( "Automatically mock your HTTP interactions to simplify and " "speed up testing" From ffc4dca502f7b129616077ed740d241e4d1e48fd Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Wed, 26 Sep 2018 00:30:11 -0700 Subject: [PATCH 26/64] ClientResponse.release isn't a coroutine Therefore it should not be one in the MockClientResponse class. https://github.com/aio-libs/aiohttp/blob/d0af887e3121d677b32339adff6540b6de01d167/aiohttp/client_reqrep.py#L832 --- vcr/stubs/aiohttp_stubs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index 38f337e..489fcf0 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -34,7 +34,7 @@ class MockClientResponse(ClientResponse): async def read(self): return self._body - async def release(self): + def release(self): pass From 0d2f49fe8ac69b27d23013b790e4efb8df8dd881 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sat, 6 Oct 2018 17:21:42 +0800 Subject: [PATCH 27/64] Fix a typo in vcr/request.py --- vcr/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/request.py b/vcr/request.py index 882db69..e268053 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -112,7 +112,7 @@ class HeadersDict(CaseInsensitiveDict): In addition, some servers sometimes send the same header more than once, and httplib *can* deal with this situation. - Futhermore, I wanted to keep the request and response cassette format as + Furthermore, I wanted to keep the request and response cassette format as similar as possible. For this reason, in cassettes I keep a dict with lists as keys, but once From 4ef5205094c886155403264dd2aa88cf53eb837a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 15 Oct 2018 02:55:59 +0100 Subject: [PATCH 28/64] support ClientResponse.text(errors=) kwarg --- tests/integration/test_aiohttp.py | 2 ++ vcr/stubs/aiohttp_stubs/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index 74aaccc..c20dc61 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -168,6 +168,8 @@ def test_aiohttp_test_client(aiohttp_client, tmpdir): assert response.status == 200 response_text = loop.run_until_complete(response.text()) assert response_text == 'hello' + response_text = loop.run_until_complete(response.text(errors='replace')) + assert response_text == 'hello' with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette: response = loop.run_until_complete(client.get(url)) diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index 489fcf0..f6525c7 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -28,8 +28,8 @@ class MockClientResponse(ClientResponse): async def json(self, *, encoding='utf-8', loads=json.loads, **kwargs): # NOQA: E999 return loads(self._body.decode(encoding)) - async def text(self, encoding='utf-8'): - return self._body.decode(encoding) + async def text(self, encoding='utf-8', errors='strict'): + return self._body.decode(encoding, errors=errors) async def read(self): return self._body From 6c166482d9f1d5deeeb6836095f110fc61f28356 Mon Sep 17 00:00:00 2001 From: jxltom Date: Wed, 14 Nov 2018 09:48:49 +0800 Subject: [PATCH 29/64] Pin yarl and multidict version for python34 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ae821ac..3c84091 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,9 @@ install_requires = [ 'six>=1.5', 'contextlib2; python_version=="2.7"', 'mock; python_version=="2.7"', - 'yarl; python_version>="3.4"', + 'yarl; python_version>"3.4"', + 'yarl<1.0.0; python_version=="3.4"', + 'multidict<4.0.0,>=2.0; python_version=="3.4"' ] excluded_packages = ["tests*"] From c3705dae9faf6b3f11d380eba8405ed4452dc907 Mon Sep 17 00:00:00 2001 From: jxltom Date: Wed, 14 Nov 2018 10:27:38 +0800 Subject: [PATCH 30/64] Fix flake8 in python3 --- vcr/serializers/jsonserializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/serializers/jsonserializer.py b/vcr/serializers/jsonserializer.py index 3378c27..000c894 100644 --- a/vcr/serializers/jsonserializer.py +++ b/vcr/serializers/jsonserializer.py @@ -25,5 +25,5 @@ def serialize(cassette_dict): original.end, original.args[-1] + error_message ) - except TypeError as original: # py3 + except TypeError: # py3 raise TypeError(error_message) From 472bc3aea1f05e920bd1ffaa33d1539c1b563ca0 Mon Sep 17 00:00:00 2001 From: Karim Hamidou Date: Thu, 20 Dec 2018 12:44:07 +0100 Subject: [PATCH 31/64] Add a `rewind` method to reset a cassette. --- tests/unit/test_cassettes.py | 11 +++++++++++ vcr/cassette.py | 3 +++ 2 files changed, 14 insertions(+) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index e719156..4541530 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -133,6 +133,17 @@ def test_cassette_all_played(): assert a.all_played +@mock.patch('vcr.cassette.requests_match', _mock_requests_match) +def test_cassette_rewound(): + a = Cassette('test') + a.append('foo', 'bar') + a.play_response('foo') + assert a.all_played + + a.rewind() + assert not a.all_played + + def test_before_record_response(): before_record_response = mock.Mock(return_value='mutated') cassette = Cassette('test', before_record_response=before_record_response) diff --git a/vcr/cassette.py b/vcr/cassette.py index 5683ddf..79d8816 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -287,6 +287,9 @@ class Cassette(object): % (self._path, request) ) + def rewind(self): + self.play_counts = collections.Counter() + def _as_dict(self): return {"requests": self.requests, "responses": self.responses} From 4e990db32eceaf2f24f273f729f5787a0db4001e Mon Sep 17 00:00:00 2001 From: David Wilemski Date: Tue, 1 Jan 2019 15:23:51 -0800 Subject: [PATCH 32/64] Fix collections.abc DeprecationWarning In versions of Python from 3.8 and forward, importing Mapping and MutableMapping from the collections module will no longer work. This change will try to import from the collections.abc module, which was added in Python 3.3, and fall back to the collections module on older versions of Python. --- vcr/util.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vcr/util.py b/vcr/util.py index 2e6effb..34185e2 100644 --- a/vcr/util.py +++ b/vcr/util.py @@ -1,13 +1,17 @@ -import collections import types +try: + from collections.abc import Mapping, MutableMapping +except ImportError: + from collections import Mapping, MutableMapping + # Shamelessly stolen from https://github.com/kennethreitz/requests/blob/master/requests/structures.py -class CaseInsensitiveDict(collections.MutableMapping): +class CaseInsensitiveDict(MutableMapping): """ A case-insensitive ``dict``-like object. Implements all methods and operations of - ``collections.MutableMapping`` as well as dict's ``copy``. Also + ``collections.abc.MutableMapping`` as well as dict's ``copy``. Also provides ``lower_items``. All keys are expected to be strings. The structure remembers the case of the last key to be set, and ``iter(instance)``, @@ -57,7 +61,7 @@ class CaseInsensitiveDict(collections.MutableMapping): ) def __eq__(self, other): - if isinstance(other, collections.Mapping): + if isinstance(other, Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented From 20e8f4ad41e97f686fe3cece38942c442c7f71eb Mon Sep 17 00:00:00 2001 From: Pi Delport Date: Mon, 4 Feb 2019 12:15:23 +0200 Subject: [PATCH 33/64] Doc typo: yml -> yaml --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index e7d6fe2..aed269d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -11,7 +11,7 @@ Usage assert 'Example domains' in response Run this test once, and VCR.py will record the HTTP request to -``fixtures/vcr_cassettes/synopsis.yml``. Run it again, and VCR.py will +``fixtures/vcr_cassettes/synopsis.yaml``. Run it again, and VCR.py will replay the response from iana.org when the http request is made. This test is now fast (no real HTTP requests are made anymore), deterministic (the test will continue to pass, even if you are offline, or iana.org From dc174c325037bb968bf3f7d541507d1be50fcd89 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 22 May 2019 17:30:45 +0300 Subject: [PATCH 34/64] Drop support for EOL Python 3.4 --- .travis.yml | 12 ------------ docs/installation.rst | 2 +- setup.py | 7 ++----- tox.ini | 2 +- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17a2546..180b1dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false before_install: openssl version env: global: @@ -18,37 +17,27 @@ matrix: - env: TOX_SUFFIX="flakes" python: 3.7 dist: xenial - sudo: true - env: TOX_SUFFIX="requests27" python: 3.7 dist: xenial - sudo: true - env: TOX_SUFFIX="httplib2" python: 3.7 dist: xenial - sudo: true - env: TOX_SUFFIX="urllib3121" python: 3.7 dist: xenial - sudo: true - env: TOX_SUFFIX="tornado4" python: 3.7 dist: xenial - sudo: true - env: TOX_SUFFIX="aiohttp" python: 3.7 dist: xenial - sudo: true allow_failures: - env: TOX_SUFFIX="boto3" - env: TOX_SUFFIX="aiohttp" python: "pypy3.5-5.9.0" - - env: TOX_SUFFIX="aiohttp" - python: 3.4 exclude: # Only run flakes on a single Python 2.x and a single 3.x - - env: TOX_SUFFIX="flakes" - python: 3.4 - env: TOX_SUFFIX="flakes" python: 3.5 - env: TOX_SUFFIX="flakes" @@ -61,7 +50,6 @@ matrix: python: pypy python: - 2.7 -- 3.4 - 3.5 - 3.6 - pypy diff --git a/docs/installation.rst b/docs/installation.rst index afec93b..8e475e0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -9,7 +9,7 @@ with pip:: Compatibility ------------- -VCR.py supports Python 2.7 and 3.4+, and +VCR.py supports Python 2.7 and 3.5+, and `pypy `__. The following HTTP libraries are supported: diff --git a/setup.py b/setup.py index 3c84091..ad54f83 100644 --- a/setup.py +++ b/setup.py @@ -28,9 +28,7 @@ install_requires = [ 'six>=1.5', 'contextlib2; python_version=="2.7"', 'mock; python_version=="2.7"', - 'yarl; python_version>"3.4"', - 'yarl<1.0.0; python_version=="3.4"', - 'multidict<4.0.0,>=2.0; python_version=="3.4"' + 'yarl; python_version>"3.5"', ] excluded_packages = ["tests*"] @@ -49,7 +47,7 @@ setup( author_email='me@kevinmccarthy.org', url='https://github.com/kevin1024/vcrpy', packages=find_packages(exclude=excluded_packages), - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', install_requires=install_requires, license='MIT', tests_require=['pytest', 'mock', 'pytest-httpbin'], @@ -61,7 +59,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', diff --git a/tox.ini b/tox.ini index bd99f42..865cd62 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py34,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} +envlist = {py27,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} [testenv:flakes] skipsdist = True From 7670e10bc2550308b8459d02b1a1513079c3489f Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 22 May 2019 17:35:15 +0300 Subject: [PATCH 35/64] Fix Flake8: E117 over-indented --- vcr/migration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index ba5ae2c..919494f 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -159,9 +159,9 @@ def main(): for (root, dirs, files) in os.walk(path) for name in files) for file_path in files: - migrated = try_migrate(file_path) - status = 'OK' if migrated else 'FAIL' - sys.stderr.write("[{}] {}\n".format(status, file_path)) + migrated = try_migrate(file_path) + status = 'OK' if migrated else 'FAIL' + sys.stderr.write("[{}] {}\n".format(status, file_path)) sys.stderr.write("Done.\n") From 8f4e089200c8d434a1311ab7334daa434688468d Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 22 May 2019 17:44:36 +0300 Subject: [PATCH 36/64] Upgrade Python syntax with pyupgrade --- tests/integration/test_proxy.py | 2 +- vcr/migration.py | 4 ++-- vcr/request.py | 2 +- vcr/stubs/__init__.py | 2 +- vcr/util.py | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 9b3c4ad..09db775 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -44,7 +44,7 @@ def proxy_server(): target=httpd.serve_forever, ) proxy_process.start() - yield 'http://{0}:{1}'.format(*httpd.server_address) + yield 'http://{}:{}'.format(*httpd.server_address) proxy_process.terminate() diff --git a/vcr/migration.py b/vcr/migration.py index 919494f..435f464 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -68,7 +68,7 @@ def _migrate(data): for item in data: req = item['request'] res = item['response'] - uri = dict((k, req.pop(k)) for k in PARTS) + uri = {k: req.pop(k) for k in PARTS} req['uri'] = build_uri(**uri) # convert headers to dict of lists headers = req['headers'] @@ -100,7 +100,7 @@ def migrate_json(in_fp, out_fp): def _list_of_tuples_to_dict(fs): - return dict((k, v) for k, v in fs[0]) + return {k: v for k, v in fs[0]} def _already_migrated(data): diff --git a/vcr/request.py b/vcr/request.py index e268053..09c83f4 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -91,7 +91,7 @@ class Request(object): 'method': self.method, 'uri': self.uri, 'body': self.body, - 'headers': dict(((k, [v]) for k, v in self.headers.items())), + 'headers': {k: [v] for k, v in self.headers.items()}, } @classmethod diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index a23a731..c01ba59 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -139,7 +139,7 @@ class VCRConnection(object): if url and not url.startswith('/'): # Then this must be a proxy request. return url - uri = "{0}://{1}{2}{3}".format( + uri = "{}://{}{}{}".format( self._protocol, self.real_connection.host, self._port_postfix(), diff --git a/vcr/util.py b/vcr/util.py index 34185e2..c7ababb 100644 --- a/vcr/util.py +++ b/vcr/util.py @@ -118,10 +118,10 @@ def auto_decorate( ) def __new__(cls, name, bases, attributes_dict): - new_attributes_dict = dict( - (attribute, maybe_decorate(attribute, value)) + new_attributes_dict = { + attribute: maybe_decorate(attribute, value) for attribute, value in attributes_dict.items() - ) + } return super(DecorateAll, cls).__new__( cls, name, bases, new_attributes_dict ) From fb84928ef610b755f516e6a5ec2491ef4f48d4eb Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Mon, 10 Jun 2019 16:57:24 -0300 Subject: [PATCH 37/64] Fix build problems on requests tests due to SSL certificate problems --- docs/changelog.rst | 2 ++ tests/integration/test_wild.py | 2 +- tox.ini | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ff6e58..42ea4eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,5 +1,7 @@ Changelog --------- +- 2.0.2 (UNRELEASED) - Drop support to python 3.4 + Fix build problems on requests tests (thanks to @dunossauro) - 2.0.1 - Fix bug when using vcrpy with python 3.4 - 2.0.0 - Support python 3.7 (fix httplib2 and urllib2, thanks @felixonmars) [#356] Fixes `before_record_response` so the original response isn't changed (thanks @kgraves) diff --git a/tests/integration/test_wild.py b/tests/integration/test_wild.py index 65c76e2..ee1eafa 100644 --- a/tests/integration/test_wild.py +++ b/tests/integration/test_wild.py @@ -59,7 +59,7 @@ def test_flickr_multipart_upload(httpbin, tmpdir): def test_flickr_should_respond_with_200(tmpdir): testfile = str(tmpdir.join('flickr.yml')) with vcr.use_cassette(testfile): - r = requests.post("http://api.flickr.com/services/upload") + r = requests.post("https://api.flickr.com/services/upload", verify=False) assert r.status_code == 200 diff --git a/tox.ini b/tox.ini index 865cd62..0f2aae1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp} +envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3121,tornado4,boto3,aiohttp} [testenv:flakes] skipsdist = True @@ -18,7 +18,7 @@ deps = pytest pytest-httpbin PyYAML - requests27: requests==2.7.0 + requests: requests>=2.22.0 httplib2: httplib2 urllib3121: urllib3==1.21.1 {py27,py35,py36,pypy}-tornado4: tornado>=4,<5 From 78a0a52bd9ac08c2a313f72bf08447439ca3c74e Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:17:39 +0200 Subject: [PATCH 38/64] fix issue with tests using localhost The error we have : SSLError: hostname '127.0.0.1' doesn't match either of 'localhost', '127.0.0.1' This is fixed by adding the `ipaddress` dependency in the tox ini. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0f2aae1..901a565 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps = pytest pytest-httpbin PyYAML + ipaddress requests: requests>=2.22.0 httplib2: httplib2 urllib3121: urllib3==1.21.1 From a53121b645107307cfbc69756a2650117f33820b Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:19:45 +0200 Subject: [PATCH 39/64] do not create tox environment with python2 and aiohttp aiohttp dependency only works with python 3.5+ --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 901a565..d0542fb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3121,tornado4,boto3,aiohttp} +envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3121,tornado4,boto3},{py35,py36,py37}-{aiohttp} [testenv:flakes] skipsdist = True From c4803dbc4d5693c8cc46fa2e93721b9ea3aabb69 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:21:02 +0200 Subject: [PATCH 40/64] httplib2 has issues validating certificate with python 2.7, disabling it --- tests/integration/test_httplib2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_httplib2.py b/tests/integration/test_httplib2.py index 4eda032..e203c1f 100644 --- a/tests/integration/test_httplib2.py +++ b/tests/integration/test_httplib2.py @@ -22,7 +22,7 @@ def http(): kwargs = { 'ca_certs': pytest_httpbin.certs.where() } - if sys.version_info[:2] == (3, 7): + if sys.version_info[:2] in [(2, 7), (3, 7)]: kwargs['disable_ssl_certificate_validation'] = True return httplib2.Http(**kwargs) From 7724b364aad3215093be6ebd593fd00333a504cb Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:21:44 +0200 Subject: [PATCH 41/64] urllib3 has a new default behavior with PoolManager object creation By default, from version urllib3=>1.25, urllib3 requires and validates certificates by default when using HTTPS (https://github.com/urllib3/urllib3/blob/master/CHANGES.rst#125-2019-04-22). Set explicitly that we deactivate certificates validation with cert_reqs=`CERT_NONE`. --- tests/integration/test_urllib3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_urllib3.py b/tests/integration/test_urllib3.py index d187f7b..6d4a93c 100644 --- a/tests/integration/test_urllib3.py +++ b/tests/integration/test_urllib3.py @@ -20,7 +20,7 @@ def verify_pool_mgr(): @pytest.fixture(scope='module') def pool_mgr(): - return urllib3.PoolManager() + return urllib3.PoolManager(cert_reqs='CERT_NONE') def test_status_code(httpbin_both, tmpdir, verify_pool_mgr): From 86586e8cd90a054402689a03eb6535e3f519491a Mon Sep 17 00:00:00 2001 From: Hugo Date: Mon, 17 Jun 2019 11:48:12 +0300 Subject: [PATCH 42/64] Remove broken Waffle badge Waffle shut down on May 16th, 2019 "due to market direction and the acquisition by Broadcom". https://help.waffle.io/articles/2801857-waffle-shutdown-tl-dr --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8cd37e9..8fcf330 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|PyPI| |Python versions| |Build Status| |Waffle Ready| |Gitter| +|PyPI| |Python versions| |Build Status| |Gitter| VCR.py ====== @@ -53,8 +53,6 @@ more details :target: https://pypi.python.org/pypi/vcrpy .. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.svg?branch=master :target: http://travis-ci.org/kevin1024/vcrpy -.. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.svg?label=ready&title=waffle - :target: https://waffle.io/kevin1024/vcrpy .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg :alt: Join the chat at https://gitter.im/kevin1024/vcrpy :target: https://gitter.im/kevin1024/vcrpy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge From 34f0417dc97cc6d0d44e1a58b90734dcffd9f315 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:51:28 +0200 Subject: [PATCH 43/64] add troubleshooting in contributing section When I ran the test suite on macOSX, I had some issues regarding SSL configuration, I have documented the error I encounter and the solution to deal with it. --- docs/contributing.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index f5ffc84..63865a6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -23,3 +23,24 @@ documentation `__ for how to set this up. I have marked the boto tests as optional in Travis so you don't have to worry about them failing if you submit a pull request. + + +Troubleshooting on MacOSX +------------------------- + +If you have this kind of error when running tox : + +.. code:: python + + __main__.ConfigurationError: Curl is configured to use SSL, but we have + not been able to determine which SSL backend it is using. Please see PycURL documentation for how to specify the SSL backend manually. + +Then you need to define some environment variables: + +.. code:: bash + + export PYCURL_SSL_LIBRARY=openssl + export LDFLAGS=-L/usr/local/opt/openssl/lib + export CPPFLAGS=-I/usr/local/opt/openssl/include + +Reference : `stackoverflow issue `__ From bdb74b9841e411d2d47af9760f00e43e5648e1f8 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Fri, 14 Jun 2019 12:42:19 +0200 Subject: [PATCH 44/64] change requests27 in travis-ci to requests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 180b1dd..4988390 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ env: - secure: LBSEg/gMj4u4Hrpo3zs6Y/1mTpd2RtcN49mZIFgTdbJ9IhpiNPqcEt647Lz94F9Eses2x2WbNuKqZKZZReY7QLbEzU1m0nN5jlaKrjcG5NR5clNABfFFyhgc0jBikyS4abAG8jc2efeaTrFuQwdoF4sE8YiVrkiVj2X5Xoi6sBk= matrix: - TOX_SUFFIX="flakes" - - TOX_SUFFIX="requests27" + - TOX_SUFFIX="requests" - TOX_SUFFIX="httplib2" - TOX_SUFFIX="boto3" - TOX_SUFFIX="urllib3121" @@ -17,7 +17,7 @@ matrix: - env: TOX_SUFFIX="flakes" python: 3.7 dist: xenial - - env: TOX_SUFFIX="requests27" + - env: TOX_SUFFIX="requests" python: 3.7 dist: xenial - env: TOX_SUFFIX="httplib2" From de244a968fd55e54dfc6546be58bf31fe6e83f09 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 11:51:00 +0200 Subject: [PATCH 45/64] add function to format the assertion message This function is used to prettify the assertion message when a matcher failed and return an assertion error. --- tests/unit/test_matchers.py | 15 +++++++++++++++ vcr/matchers.py | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 9889b4f..f7cfb9f 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -157,3 +157,18 @@ def test_metchers(): assert_matcher('port') assert_matcher('path') assert_matcher('query') + + +def test_get_assertion_message(): + assert matchers.get_assertion_message(None) == "" + assert matchers.get_assertion_message("") == "" + + +def test_get_assertion_message_with_details(): + assertion_msg = "q1=1 != q2=1" + expected = ( + "--------------- DETAILS ---------------\n" + "{}\n" + "----------------------------------------\n".format(assertion_msg) + ) + assert matchers.get_assertion_message(assertion_msg) == expected diff --git a/vcr/matchers.py b/vcr/matchers.py index ad04a45..784cfe4 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -99,3 +99,23 @@ def requests_match(r1, r2, matchers): matches = [(m(r1, r2), m) for m in matchers] _log_matches(r1, r2, matches) return all(m[0] for m in matches) + + +def get_assertion_message(assertion_details, **format_options): + """ + Get a detailed message about the failing matcher. + """ + msg = "" + if assertion_details: + separator = format_options.get("separator", "-") + title = format_options.get("title", " DETAILS ") + nb_separator = format_options.get("nb_separator", 40) + first_title_line = ( + separator * ((nb_separator - len(title)) // 2) + + title + + separator * ((nb_separator - len(title)) // 2) + ) + msg += "{}\n{}\n{}\n".format( + first_title_line, str(assertion_details), separator * nb_separator + ) + return msg From 940dec1dd68cc9902b01905a4374035164ef9c76 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 11:57:54 +0200 Subject: [PATCH 46/64] add private function to evaluate a matcher A matcher can now return other results than a boolean : - An AssertionError exception meaning that the matcher failed, with the exception we get the assertion failure message. - None, in case we do an assert in the matcher, meaning that the assertion has passed, the matcher is considered as a success then. - Boolean that indicates if a matcher failed or not. If there is no match, a boolean does not give any clue what it is the differences compared to the assertion. --- tests/unit/test_matchers.py | 48 +++++++++++++++++++++++++++---------- vcr/matchers.py | 17 +++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index f7cfb9f..1640d42 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -143,20 +143,44 @@ def test_query_matcher(): req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {}) assert matchers.query(req1, req2) - req1 = request.Request('GET', 'http://host.com/?a=b&a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com/?a=b&c=d&a=b', '', {}) - req3 = request.Request('GET', 'http://host.com/?c=d&a=b&a=b', '', {}) - assert matchers.query(req1, req2) - assert matchers.query(req1, req3) +def test_evaluate_matcher_does_match(): + def bool_matcher(r1, r2): + return True + + def assertion_matcher(r1, r2): + assert 1 == 1 + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is True + assert assertion_msg is None -def test_metchers(): - assert_matcher('method') - assert_matcher('scheme') - assert_matcher('host') - assert_matcher('port') - assert_matcher('path') - assert_matcher('query') +def test_evaluate_matcher_does_not_match(): + def bool_matcher(r1, r2): + return False + + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError() + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is False + assert not assertion_msg + + +def test_evaluate_matcher_does_not_match_with_assert_message(): + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError("Failing matcher") + + r1, r2 = None, None + match, assertion_msg = matchers._evaluate_matcher(assertion_matcher, r1, r2) + assert match is False + assert assertion_msg == "Failing matcher" def test_get_assertion_message(): diff --git a/vcr/matchers.py b/vcr/matchers.py index 784cfe4..5885058 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -101,6 +101,23 @@ def requests_match(r1, r2, matchers): return all(m[0] for m in matches) +def _evaluate_matcher(matcher_function, *args): + """ + Evaluate the result of a given matcher as a boolean with an assertion error message if any. + It handles two types of matcher : + - a matcher returning a boolean value. + - a matcher that only makes an assert, returning None or raises an assertion error. + """ + assertion_message = None + try: + match = matcher_function(*args) + match = True if match is None else match + except AssertionError as e: + match = False + assertion_message = str(e) + return match, assertion_message + + def get_assertion_message(assertion_details, **format_options): """ Get a detailed message about the failing matcher. From 46f5b8a1879b730a33659b73ace70e8c99b17de6 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 12:07:04 +0200 Subject: [PATCH 47/64] add function to get the comparaison result of two request with a list of matchers The function returns two list: - the first one is the list of matchers names that have succeeded. - the second is a list of tuples with the failed matchers names and the related assertion message like this ("matcher_name", "assertion_message"). If the second list is empty, it means that all the matchers have passed. --- tests/unit/test_matchers.py | 34 ++++++++++++++++++++++++++++++++++ vcr/matchers.py | 18 ++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 1640d42..5ddb583 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -196,3 +196,37 @@ def test_get_assertion_message_with_details(): "----------------------------------------\n".format(assertion_msg) ) assert matchers.get_assertion_message(assertion_msg) == expected + + +@pytest.mark.parametrize( + "r1, r2, expected_successes, expected_failures", + [ + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("GET", "http://host.com/p?a=b", "", {}), + ["method", "path"], + [], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/p?a=b", "", {}), + ["path"], + ["method"], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/path?a=b", "", {}), + [], + ["method", "path"], + ), + ], +) +def test_get_matchers_results(r1, r2, expected_successes, expected_failures): + successes, failures = matchers.get_matchers_results( + r1, r2, [matchers.method, matchers.path] + ) + assert successes == expected_successes + assert len(failures) == len(expected_failures) + for i, expected_failure in enumerate(expected_failures): + assert failures[i][0] == expected_failure + assert failures[i][1] is not None diff --git a/vcr/matchers.py b/vcr/matchers.py index 5885058..ede73fb 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -118,6 +118,24 @@ def _evaluate_matcher(matcher_function, *args): return match, assertion_message +def get_matchers_results(r1, r2, matchers): + """ + Get the comparison results of two requests as two list. + The first returned list represents the matchers names that passed. + The second list is the failed matchers as a string with failed assertion details if any. + """ + matches_success, matches_fails = [], [] + for m in matchers: + matcher_name = m.__name__ + match, assertion_message = _evaluate_matcher(m, r1, r2) + if match: + matches_success.append(matcher_name) + else: + assertion_message = get_assertion_message(assertion_message) + matches_fails.append((matcher_name, assertion_message)) + return matches_success, matches_fails + + def get_assertion_message(assertion_details, **format_options): """ Get a detailed message about the failing matcher. From 0a01f0fb5133582d9d0e931e98fe9ea0661c77bb Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 12:20:30 +0200 Subject: [PATCH 48/64] change all the matchers with an assert statement and refactor the requests_match function In order to use the new assert mechanism that returns explicit assertion failure message, all the default matchers does not return a boolean, but only do an assert statement with a basic assertion message (value_1 != value_2). The requests_match function has been refactored to use the 'get_matchers_results' function in order to have explicit failures that are logged if any. Many unit tests have been changed as the matchers does not return a boolean value anymore. Note: Only the matchers "body" and "raw_body" does not have an assertion message, the body values might be big and not useful to be display to spot the differences. --- tests/unit/test_matchers.py | 63 +++++++++++++++++++++++++++++-------- vcr/matchers.py | 60 +++++++++++++++++------------------ 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 5ddb583..604b650 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -1,4 +1,5 @@ import itertools +from vcr.compat import mock import pytest @@ -21,20 +22,22 @@ REQUESTS = { def assert_matcher(matcher_name): matcher = getattr(matchers, matcher_name) for k1, k2 in itertools.permutations(REQUESTS, 2): - matched = matcher(REQUESTS[k1], REQUESTS[k2]) - if matcher_name in {k1, k2}: - assert not matched + expecting_assertion_error = matcher_name in {k1, k2} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matcher(REQUESTS[k1], REQUESTS[k2]) else: - assert matched + assert matcher(REQUESTS[k1], REQUESTS[k2]) is None def test_uri_matcher(): for k1, k2 in itertools.permutations(REQUESTS, 2): - matched = matchers.uri(REQUESTS[k1], REQUESTS[k2]) - if {k1, k2} != {'base', 'method'}: - assert not matched + expecting_assertion_error = {k1, k2} != {"base", "method"} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matchers.uri(REQUESTS[k1], REQUESTS[k2]) else: - assert matched + assert matchers.uri(REQUESTS[k1], REQUESTS[k2]) is None req1_body = (b"test" @@ -107,7 +110,7 @@ req2_body = (b"test" ) ]) def test_body_matcher_does_match(r1, r2): - assert matchers.body(r1, r2) + assert matchers.body(r1, r2) is None @pytest.mark.parametrize("r1, r2", [ @@ -135,13 +138,30 @@ def test_body_matcher_does_match(r1, r2): ) ]) def test_body_match_does_not_match(r1, r2): - assert not matchers.body(r1, r2) + with pytest.raises(AssertionError): + matchers.body(r1, r2) def test_query_matcher(): - req1 = request.Request('GET', 'http://host.com/?a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {}) - assert matchers.query(req1, req2) + req1 = request.Request("GET", "http://host.com/?a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?c=d&a=b", "", {}) + assert matchers.query(req1, req2) is None + + req1 = request.Request("GET", "http://host.com/?a=b&a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?a=b&c=d&a=b", "", {}) + req3 = request.Request("GET", "http://host.com/?c=d&a=b&a=b", "", {}) + assert matchers.query(req1, req2) is None + assert matchers.query(req1, req3) is None + + +def test_matchers(): + assert_matcher("method") + assert_matcher("scheme") + assert_matcher("host") + assert_matcher("port") + assert_matcher("path") + assert_matcher("query") + def test_evaluate_matcher_does_match(): def bool_matcher(r1, r2): @@ -230,3 +250,20 @@ def test_get_matchers_results(r1, r2, expected_successes, expected_failures): for i, expected_failure in enumerate(expected_failures): assert failures[i][0] == expected_failure assert failures[i][1] is not None + + +@mock.patch("vcr.matchers.get_matchers_results") +@pytest.mark.parametrize( + "successes, failures, expected_match", + [ + (["method", "path"], [], True), + (["method"], ["path"], False), + ([], ["method", "path"], False), + ], +) +def test_requests_match(mock_get_matchers_results, successes, failures, expected_match): + mock_get_matchers_results.return_value = (successes, failures) + r1 = request.Request("GET", "http://host.com/p?a=b", "", {}) + r2 = request.Request("GET", "http://host.com/p?a=b", "", {}) + match = matchers.requests_match(r1, r2, [matchers.method, matchers.path]) + assert match is expected_match diff --git a/vcr/matchers.py b/vcr/matchers.py index ede73fb..1714018 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -8,35 +8,47 @@ log = logging.getLogger(__name__) def method(r1, r2): - return r1.method == r2.method + assert r1.method == r2.method, "{} != {}".format(r1.method, r2.method) def uri(r1, r2): - return r1.uri == r2.uri + assert r1.uri == r2.uri, "{} != {}".format(r1.uri, r2.uri) def host(r1, r2): - return r1.host == r2.host + assert r1.host == r2.host, "{} != {}".format(r1.host, r2.host) def scheme(r1, r2): - return r1.scheme == r2.scheme + assert r1.scheme == r2.scheme, "{} != {}".format(r1.scheme, r2.scheme) def port(r1, r2): - return r1.port == r2.port + assert r1.port == r2.port, "{} != {}".format(r1.port, r2.port) def path(r1, r2): - return r1.path == r2.path + assert r1.path == r2.path, "{} != {}".format(r1.path, r2.path) def query(r1, r2): - return r1.query == r2.query + assert r1.query == r2.query, "{} != {}".format(r1.query, r2.query) def raw_body(r1, r2): - return read_body(r1) == read_body(r2) + assert read_body(r1) == read_body(r2) + + +def body(r1, r2): + transformer = _get_transformer(r1) + r2_transformer = _get_transformer(r2) + if transformer != r2_transformer: + transformer = _identity + assert transformer(read_body(r1)) == transformer(read_body(r2)) + + +def headers(r1, r2): + assert r1.headers == r2.headers, "{} != {}".format(r1.headers, r2.headers) def _header_checker(value, header='Content-Type'): @@ -74,31 +86,15 @@ def _get_transformer(request): return _identity -def body(r1, r2): - transformer = _get_transformer(r1) - r2_transformer = _get_transformer(r2) - if transformer != r2_transformer: - transformer = _identity - return transformer(read_body(r1)) == transformer(read_body(r2)) - - -def headers(r1, r2): - return r1.headers == r2.headers - - -def _log_matches(r1, r2, matches): - differences = [m for m in matches if not m[0]] - if differences: - log.debug( - "Requests {} and {} differ according to " - "the following matchers: {}".format(r1, r2, differences) - ) - - def requests_match(r1, r2, matchers): - matches = [(m(r1, r2), m) for m in matchers] - _log_matches(r1, r2, matches) - return all(m[0] for m in matches) + successes, failures = get_matchers_results(r1, r2, matchers) + if failures: + log.debug( + "Requests {} and {} differ.\n" + "Failure details:\n" + "{}".format(r1, r2, failures) + ) + return len(failures) == 0 def _evaluate_matcher(matcher_function, *args): From 396c4354e8482956f9d779bf2a4660f18b8b573d Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 13:17:15 +0200 Subject: [PATCH 49/64] add cassette's method to find the most similar request(s) of a request This method get the requests in the cassettes with the most matchers that succeeds. --- tests/unit/test_cassettes.py | 48 ++++++++++++++++++++++++++++++++++++ vcr/cassette.py | 34 ++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 4541530..01d8c70 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -317,3 +317,51 @@ def test_use_as_decorator_on_generator(): yield 2 assert list(test_function()) == [1, 2] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_one_similar_request(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ([], [("method", "failed : method"), ("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [(2, ["method", "path"], [("query", "failed : query")])] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_no_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_many_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method", "path"], [("query", "failed : query")]), + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [ + (1, ["method", "path"], [("query", "failed : query")]), + (3, ["method", "path"], [("query", "failed : query")]) + ] diff --git a/vcr/cassette.py b/vcr/cassette.py index 79d8816..7ba092c 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -8,7 +8,7 @@ import wrapt from .compat import contextlib from .errors import UnhandledHTTPRequestError -from .matchers import requests_match, uri, method +from .matchers import requests_match, uri, method, get_matchers_results from .patch import CassettePatcherBuilder from .serializers import yamlserializer from .persisters.filesystem import FilesystemPersister @@ -290,6 +290,38 @@ class Cassette(object): def rewind(self): self.play_counts = collections.Counter() + def find_requests_with_most_matches(self, request): + """ + Get the most similar request(s) stored in the cassette + of a given request as a list of tuples like this: + - the request object + - the successful matchers as string + - the failed matchers and the related assertion message with the difference details as strings tuple + + This is useful when a request failed to be found, + we can get the similar request(s) in order to know what have changed in the request parts. + """ + best_matches = [] + request = self._before_record_request(request) + for index, (stored_request, response) in enumerate(self.data): + successes, fails = get_matchers_results(request, stored_request, self._match_on) + best_matches.append((len(successes), stored_request, successes, fails)) + best_matches.sort(key=lambda t: t[0], reverse=True) + # Get the first best matches (multiple if equal matches) + final_best_matches = [] + previous_nb_success = best_matches[0][0] + for best_match in best_matches: + nb_success = best_match[0] + # Do not keep matches that have 0 successes, + # it means that the request is totally different from + # the ones stored in the cassette + if nb_success < 1 or previous_nb_success != nb_success: + break + previous_nb_success = nb_success + final_best_matches.append(best_match[1:]) + + return final_best_matches + def _as_dict(self): return {"requests": self.requests, "responses": self.responses} From 28d9899b9b144aac1d83a7ab619402f5e010628e Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Wed, 12 Jun 2019 13:24:13 +0200 Subject: [PATCH 50/64] refactor the 'CannotOverwriteExistingCassetteException' exception, building a more detailed message The 'CannotOverwriteExistingCassetteException' exception now takes two kwargs, cassette and failed requests, in order to get the request(s) in the cassettes with the less differences and put those details in the exception message. --- vcr/errors.py | 27 ++++++++++++++++++++++++++- vcr/stubs/__init__.py | 7 ++----- vcr/stubs/tornado_stubs.py | 6 ++---- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/vcr/errors.py b/vcr/errors.py index bdc9701..f762e8c 100644 --- a/vcr/errors.py +++ b/vcr/errors.py @@ -1,5 +1,30 @@ class CannotOverwriteExistingCassetteException(Exception): - pass + def __init__(self, *args, **kwargs): + message = self._get_message(kwargs["cassette"], kwargs["failed_request"]) + super(CannotOverwriteExistingCassetteException, self).__init__(message) + + def _get_message(self, cassette, failed_request): + """Get the final message related to the exception""" + # Get the similar requests in the cassette that + # have match the most with the request. + best_matches = cassette.find_requests_with_most_matches(failed_request) + # Build a comprehensible message to put in the exception. + best_matches_msg = "" + for best_match in best_matches: + request, _, failed_matchers_assertion_msgs = best_match + best_matches_msg += "Similar request found : (%r).\n" % request + for failed_matcher, assertion_msg in failed_matchers_assertion_msgs: + best_matches_msg += "Matcher failed : %s\n" "%s\n" % ( + failed_matcher, + assertion_msg, + ) + return ( + "Can't overwrite existing cassette (%r) in " + "your current record mode (%r).\n" + "No match for the request (%r) was found.\n" + "%s" + % (cassette._path, cassette.record_mode, failed_request, best_matches_msg) + ) class UnhandledHTTPRequestError(KeyError): diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index c01ba59..05490bb 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -230,11 +230,8 @@ class VCRConnection(object): self._vcr_request ): raise CannotOverwriteExistingCassetteException( - "No match for the request (%r) was found. " - "Can't overwrite existing cassette (%r) in " - "your current record mode (%r)." - % (self._vcr_request, self.cassette._path, - self.cassette.record_mode) + cassette=self.cassette, + failed_request=self._vcr_request ) # Otherwise, we should send the request, then get the response diff --git a/vcr/stubs/tornado_stubs.py b/vcr/stubs/tornado_stubs.py index b675065..5beea4b 100644 --- a/vcr/stubs/tornado_stubs.py +++ b/vcr/stubs/tornado_stubs.py @@ -75,10 +75,8 @@ def vcr_fetch_impl(cassette, real_fetch_impl): request, 599, error=CannotOverwriteExistingCassetteException( - "No match for the request (%r) was found. " - "Can't overwrite existing cassette (%r) in " - "your current record mode (%r)." - % (vcr_request, cassette._path, cassette.record_mode) + cassette=cassette, + failed_request=vcr_request ), request_time=self.io_loop.time() - request.start_time, ) From f414e04f49b7544b40a1727bc5bd2a7ccc823cff Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:14:11 +0200 Subject: [PATCH 51/64] update the documentation of custom matchers Add documentation on creating a matcher with an `assert` statement that provides assertion messages in case of failures. --- docs/advanced.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 198a3d2..ed5aa08 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -97,8 +97,12 @@ Create your own method with the following signature def my_matcher(r1, r2): -Your method receives the two requests and must return ``True`` if they -match, ``False`` if they don't. +Your method receives the two requests and can return : + +- Use an ``assert`` statement in the matcher, then we have ``None`` if they match, raise an `AssertionError`` if they don't. +- A boolean, ``True`` if they match, ``False`` if they don't. + +Note : You should use an ``assert`` statement in order to have feedback when a matcher is failing. Finally, register your method with VCR to use your new request matcher. @@ -107,7 +111,7 @@ Finally, register your method with VCR to use your new request matcher. import vcr def jurassic_matcher(r1, r2): - return r1.uri == r2.uri and 'JURASSIC PARK' in r1.body + assert r1.uri == r2.uri and 'JURASSIC PARK' in r1.body my_vcr = vcr.VCR() my_vcr.register_matcher('jurassic', jurassic_matcher) From b203fd4113fb865bd02e39c862e21e90c73a896d Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Thu, 13 Jun 2019 15:15:37 +0200 Subject: [PATCH 52/64] add reference to pytest-vcr plugin in the documentation --- docs/usage.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index e7d6fe2..e4223d2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -95,3 +95,9 @@ Unittest Integration While it's possible to use the context manager or decorator forms with unittest, there's also a ``VCRTestCase`` provided separately by `vcrpy-unittest `__. + +Pytest Integration +------------------ + +A Pytest plugin is available here : `pytest-vcr +`__. From 09ed0e911e530e9b907ac92f2892248b6af245fa Mon Sep 17 00:00:00 2001 From: Josh Peak Date: Fri, 28 Jun 2019 11:29:12 +1000 Subject: [PATCH 53/64] Add cassette and failed request as properties of thrown CannotOverwriteCassetteException --- vcr/errors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vcr/errors.py b/vcr/errors.py index f762e8c..bdccaca 100644 --- a/vcr/errors.py +++ b/vcr/errors.py @@ -1,5 +1,7 @@ class CannotOverwriteExistingCassetteException(Exception): def __init__(self, *args, **kwargs): + self.cassette = kwargs["cassette"] + self.failed_request = kwargs["failed_request"] message = self._get_message(kwargs["cassette"], kwargs["failed_request"]) super(CannotOverwriteExistingCassetteException, self).__init__(message) From f8e8b857901fe2decb7cdce5ffa339bd12a1292f Mon Sep 17 00:00:00 2001 From: Josh Peak Date: Fri, 28 Jun 2019 13:29:55 +1000 Subject: [PATCH 54/64] Address testing with tox on windows documentation --- docs/contributing.rst | 5 ++++- runtests.sh | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 63865a6..a2de992 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,7 +11,10 @@ yourself using `py.test `__ and all environments VCR.py supports. The test suite is pretty big and slow, but you can tell tox to only run specific tests like this:: - tox -e py27requests -- -v -k "'test_status_code or test_gzip'" + tox -e {pyNN}-{HTTP_LIBRARY} -- + + tox -e py27-requests -- -v -k "'test_status_code or test_gzip'" + tox -e py37-requests -- -v --last-failed This will run only tests that look like ``test_status_code`` or ``test_gzip`` in the test suite, and only in the python 2.7 environment diff --git a/runtests.sh b/runtests.sh index d6718bc..99cc970 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,3 +1,7 @@ #!/bin/bash +# https://blog.ionelmc.ro/2015/04/14/tox-tricks-and-patterns/#when-it-inevitably-leads-to-shell-scripts +# If you are getting an INVOCATION ERROR for this script then there is +# a good chance you are running on Windows. +# You can and should use WSL for running tox on Windows when it calls bash scripts. REQUESTS_CA_BUNDLE=`python -m pytest_httpbin.certs` py.test $* From 1b6f304421e6aef6a56c980ae69d5fd5b3cbe578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steinn=20Eldj=C3=A1rn=20Sigur=C3=B0arson?= Date: Tue, 4 Jun 2019 14:26:02 +0000 Subject: [PATCH 55/64] Add .content to MockClientResponse so code which uses aiohttp request content streams directly can be tested --- vcr/stubs/aiohttp_stubs/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index f6525c7..8e03907 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -5,12 +5,16 @@ import asyncio import functools import json -from aiohttp import ClientResponse +from aiohttp import ClientResponse, streams from yarl import URL from vcr.request import Request +class MockStream(asyncio.StreamReader, streams.AsyncStreamReaderMixin): + pass + + class MockClientResponse(ClientResponse): def __init__(self, method, url): super().__init__( @@ -37,6 +41,13 @@ class MockClientResponse(ClientResponse): def release(self): pass + @property + def content(self): + s = MockStream() + s.feed_data(self._body) + s.feed_eof() + return s + def vcr_request(cassette, real_request): @functools.wraps(real_request) From cc55ef5b357af58a7803b86dec87dbe69b0ff6db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steinn=20Eldj=C3=A1rn=20Sigur=C3=B0arson?= Date: Tue, 4 Jun 2019 14:27:13 +0000 Subject: [PATCH 56/64] Add test for MockStream on aiohttp --- tests/integration/test_aiohttp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index c20dc61..c2145af 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -93,6 +93,24 @@ def test_binary(tmpdir, scheme): assert cassette.play_count == 1 +@pytest.mark.asyncio +async def test_stream(tmpdir, scheme): + url = scheme + '://httpbin.org/get' + headers = {'Content-Type': 'application/json'} + + with vcr.use_cassette(str(tmpdir.join('stream.yaml'))): + async with aiohttp.ClientSession() as session: + resp = await session.get(url, headers=headers) + body = await resp.read() # do not use stream interface here, the stream seems exhausted by vcr + + with vcr.use_cassette(str(tmpdir.join('stream.yaml'))) as cassette: + async with aiohttp.ClientSession() as session: + cassette_resp = await session.get(url, headers=headers) + cassette_body = await cassette_resp.content.read() + assert cassette_body == body + assert cassette.play_count == 1 + + def test_post(tmpdir, scheme): data = {'key1': 'value1', 'key2': 'value2'} url = scheme + '://httpbin.org/post' From 6c41b8b723497f8736bf1b96003d650ee5feb078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steinn=20Eldj=C3=A1rn=20Sigur=C3=B0arson?= Date: Fri, 28 Jun 2019 13:30:14 +0000 Subject: [PATCH 57/64] Use built-in helpers to avoid use of async keywords which cause syntax errors on 2.x --- tests/integration/test_aiohttp.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index c2145af..58b9a62 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -93,20 +93,15 @@ def test_binary(tmpdir, scheme): assert cassette.play_count == 1 -@pytest.mark.asyncio -async def test_stream(tmpdir, scheme): +def test_stream(tmpdir, scheme): url = scheme + '://httpbin.org/get' headers = {'Content-Type': 'application/json'} with vcr.use_cassette(str(tmpdir.join('stream.yaml'))): - async with aiohttp.ClientSession() as session: - resp = await session.get(url, headers=headers) - body = await resp.read() # do not use stream interface here, the stream seems exhausted by vcr + resp, body = get(url, output='raw') # XXX: headers? with vcr.use_cassette(str(tmpdir.join('stream.yaml'))) as cassette: - async with aiohttp.ClientSession() as session: - cassette_resp = await session.get(url, headers=headers) - cassette_body = await cassette_resp.content.read() + cassette_resp, cassette_body = get(url, output='raw') assert cassette_body == body assert cassette.play_count == 1 From 67b03b45c338143d0e1496dc0c48046ca000b8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steinn=20Eldj=C3=A1rn=20Sigur=C3=B0arson?= Date: Fri, 28 Jun 2019 13:34:48 +0000 Subject: [PATCH 58/64] Add output option to use response.content stream --- tests/integration/aiohttp_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/aiohttp_utils.py b/tests/integration/aiohttp_utils.py index 6f6f15e..c8c1b0b 100644 --- a/tests/integration/aiohttp_utils.py +++ b/tests/integration/aiohttp_utils.py @@ -17,6 +17,8 @@ async def aiohttp_request(loop, method, url, output='text', encoding='utf-8', co content = await response.json(encoding=encoding, content_type=content_type) elif output == 'raw': content = await response.read() + elif output == 'stream': + content = await response.content.read() response_ctx._resp.close() await session.close() From e8a9a65befd4c99634da369628da5104ff27b36d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steinn=20Eldj=C3=A1rn=20Sigur=C3=B0arson?= Date: Fri, 28 Jun 2019 13:35:05 +0000 Subject: [PATCH 59/64] Make test use stream output to test MockStream interface --- tests/integration/test_aiohttp.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index 58b9a62..a02c982 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -95,13 +95,12 @@ def test_binary(tmpdir, scheme): def test_stream(tmpdir, scheme): url = scheme + '://httpbin.org/get' - headers = {'Content-Type': 'application/json'} with vcr.use_cassette(str(tmpdir.join('stream.yaml'))): - resp, body = get(url, output='raw') # XXX: headers? + resp, body = get(url, output='raw') # Do not use stream here, as the stream is exhausted by vcr with vcr.use_cassette(str(tmpdir.join('stream.yaml'))) as cassette: - cassette_resp, cassette_body = get(url, output='raw') + cassette_resp, cassette_body = get(url, output='stream') assert cassette_body == body assert cassette.play_count == 1 From d4b706334c4aab51ed7047213f87c857d2281009 Mon Sep 17 00:00:00 2001 From: "James E. King III" Date: Sat, 22 Dec 2018 17:21:25 +0000 Subject: [PATCH 60/64] properly handle tunnel connect uri generation broken in #389 --- tests/unit/test_request.py | 25 ++++++++++++++++--------- vcr/request.py | 5 ++++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index dfd68d7..00793f8 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -3,9 +3,13 @@ import pytest from vcr.request import Request, HeadersDict -def test_str(): - req = Request('GET', 'http://www.google.com/', '', {}) - str(req) == '' +@pytest.mark.parametrize("method, uri, expected_str", [ + ('GET', 'http://www.google.com/', ''), + ('OPTIONS', '*', ''), + ('CONNECT', 'host.some.where:1234', '') +]) +def test_str(method, uri, expected_str): + assert str(Request(method, uri, '', {})) == expected_str def test_headers(): @@ -29,18 +33,21 @@ def test_add_header_deprecated(): ('https://go.com/', 443), ('https://go.com:443/', 443), ('https://go.com:3000/', 3000), + ('*', None) ]) def test_port(uri, expected_port): req = Request('GET', uri, '', {}) assert req.port == expected_port -def test_uri(): - req = Request('GET', 'http://go.com/', '', {}) - assert req.uri == 'http://go.com/' - - req = Request('GET', 'http://go.com:80/', '', {}) - assert req.uri == 'http://go.com:80/' +@pytest.mark.parametrize("method, uri", [ + ('GET', 'http://go.com/'), + ('GET', 'http://go.com:80/'), + ('CONNECT', 'localhost:1234'), + ('OPTIONS', '*') +]) +def test_uri(method, uri): + assert Request(method, uri, '', {}).uri == uri def test_HeadersDict(): diff --git a/vcr/request.py b/vcr/request.py index 09c83f4..b4b3f71 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -58,7 +58,10 @@ class Request(object): parse_uri = urlparse(self.uri) port = parse_uri.port if port is None: - port = {'https': 443, 'http': 80}[parse_uri.scheme] + try: + port = {'https': 443, 'http': 80}[parse_uri.scheme] + except KeyError: + pass return port @property From 95c7898b654f01f21040e447f07196bf1cc4a1bb Mon Sep 17 00:00:00 2001 From: Rishab Malik Date: Thu, 13 Jun 2019 16:46:17 -0400 Subject: [PATCH 61/64] [DEV-5365][DEV-5367] VCR responses don't work with Biopython on Python 3 (#1) * added IOBase methods and fp pointer * regression test --- tests/unit/test_response.py | 71 +++++++++++++++++++++++++++++++++++++ vcr/stubs/__init__.py | 27 +++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 4ba6dcc..01411ac 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -1,4 +1,6 @@ # coding: UTF-8 +import io + from vcr.stubs import VCRHTTPResponse @@ -66,3 +68,72 @@ def test_response_headers_should_have_correct_values(): assert response.headers.get('content-length') == "10806" assert response.headers.get('date') == "Fri, 24 Oct 2014 18:35:37 GMT" + + +def test_response_parses_correctly_and_fp_attribute_error_is_not_thrown(): + """ + Regression test for https://github.com/kevin1024/vcrpy/issues/440 + :return: + """ + recorded_response = { + "status": { + "message": "OK", + "code": 200 + }, + "headers": { + "content-length": ["0"], + "server": ["gunicorn/18.0"], + "connection": ["Close"], + "access-control-allow-credentials": ["true"], + "date": ["Fri, 24 Oct 2014 18:35:37 GMT"], + "access-control-allow-origin": ["*"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": { + "string": b"\nPMID- 19416910\nOWN - NLM\nSTAT- MEDLINE\nDA - 20090513\nDCOM- 20090622\nLR - " + b"20141209\nIS - 1091-6490 (Electronic)\nIS - 0027-8424 (Linking)\nVI - 106\nIP - " + b"19\nDP - 2009 May 12\nTI - Genetic dissection of histone deacetylase requirement in " + b"tumor cells.\nPG - 7751-5\nLID - 10.1073/pnas.0903139106 [doi]\nAB - Histone " + b"deacetylase inhibitors (HDACi) represent a new group of drugs currently\n being " + b"tested in a wide variety of clinical applications. They are especially\n effective " + b"in preclinical models of cancer where they show antiproliferative\n action in many " + b"different types of cancer cells. Recently, the first HDACi was\n approved for the " + b"treatment of cutaneous T cell lymphomas. Most HDACi currently in\n clinical " + b"development act by unspecifically interfering with the enzymatic\n activity of all " + b"class I HDACs (HDAC1, 2, 3, and 8), and it is widely believed\n that the development " + b"of isoform-specific HDACi could lead to better therapeutic\n efficacy. The " + b"contribution of the individual class I HDACs to different disease\n states, however, " + b"has so far not been fully elucidated. Here, we use a genetic\n approach to dissect " + b"the involvement of the different class I HDACs in tumor\n cells. We show that " + b"deletion of a single HDAC is not sufficient to induce cell\n death, but that HDAC1 " + b"and 2 play redundant and essential roles in tumor cell\n survival. Their deletion " + b"leads to nuclear bridging, nuclear fragmentation, and\n mitotic catastrophe, " + b"mirroring the effects of HDACi on cancer cells. These\n findings suggest that " + b"pharmacological inhibition of HDAC1 and 2 may be sufficient\n for anticancer " + b"activity, providing an experimental framework for the development \n of " + b"isoform-specific HDAC inhibitors.\nFAU - Haberland, Michael\nAU - Haberland M\nAD - " + b"Department of Molecular Biology, University of Texas Southwestern Medical Center," + b"\n 5323 Harry Hines Boulevard, Dallas, TX 75390-9148, USA.\nFAU - Johnson, " + b"Aaron\nAU - Johnson A\nFAU - Mokalled, Mayssa H\nAU - Mokalled MH\nFAU - Montgomery, " + b"Rusty L\nAU - Montgomery RL\nFAU - Olson, Eric N\nAU - Olson EN\nLA - eng\nPT - " + b"Journal Article\nPT - Research Support, N.I.H., Extramural\nPT - Research Support, " + b"Non-U.S. Gov't\nDEP - 20090429\nPL - United States\nTA - Proc Natl Acad Sci U S A\nJT " + b"- Proceedings of the National Academy of Sciences of the United States of America\nJID - " + b"7505876\nRN - 0 (Antineoplastic Agents)\nRN - 0 (Histone Deacetylase Inhibitors)\nRN - " + b"0 (Protein Isoforms)\nRN - EC 3.5.1.98 (Histone Deacetylases)\nSB - IM\nMH - " + b"Animals\nMH - Antineoplastic Agents/pharmacology\nMH - Cell Death\nMH - Cell Line, " + b"Tumor\nMH - Cell Survival\nMH - Gene Expression Regulation, Enzymologic\nMH - Gene " + b"Expression Regulation, Neoplastic\nMH - Histone Deacetylase Inhibitors\nMH - Histone " + b"Deacetylases/*genetics/*physiology\nMH - Humans\nMH - Mice\nMH - Mice, Nude\nMH - " + b"Models, Genetic\nMH - Neoplasm Transplantation\nMH - Neoplasms/metabolism\nMH - " + b"Protein Isoforms\nPMC - PMC2683118\nOID - NLM: PMC2683118\nEDAT- 2009/05/07 09:00\nMHDA- " + b"2009/06/23 09:00\nCRDT- 2009/05/07 09:00\nAID - 0903139106 [pii]\nAID - " + b"10.1073/pnas.0903139106 [doi]\nPST - ppublish\nSO - Proc Natl Acad Sci U S A. 2009 May " + b"12;106(19):7751-5. doi:\n 10.1073/pnas.0903139106. Epub 2009 Apr 29.\n " + } + } + vcr_response = VCRHTTPResponse(recorded_response) + handle = io.TextIOWrapper(io.BufferedReader(vcr_response), encoding='utf-8') + handle = iter(handle) + articles = [line for line in handle] + assert len(articles) > 1 diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 05490bb..118bc22 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -60,9 +60,10 @@ def serialize_headers(response): class VCRHTTPResponse(HTTPResponse): """ - Stub reponse class that gets returned instead of a HTTPResponse + Stub response class that gets returned instead of a HTTPResponse """ def __init__(self, recorded_response): + self.fp = None self.recorded_response = recorded_response self.reason = recorded_response['status']['message'] self.status = self.code = recorded_response['status']['code'] @@ -93,9 +94,30 @@ class VCRHTTPResponse(HTTPResponse): def read(self, *args, **kwargs): return self._content.read(*args, **kwargs) + def readall(self): + return self._content.readall() + + def readinto(self, *args, **kwargs): + return self._content.readinto(*args, **kwargs) + def readline(self, *args, **kwargs): return self._content.readline(*args, **kwargs) + def readlines(self, *args, **kwargs): + return self._content.readlines(*args, **kwargs) + + def seekable(self): + return self._content.seekable() + + def tell(self): + return self._content.tell() + + def isatty(self): + return self._content.isatty() + + def seek(self, *args, **kwargs): + return self._content.seek(*args, **kwargs) + def close(self): self._closed = True return True @@ -121,6 +143,9 @@ class VCRHTTPResponse(HTTPResponse): else: return default + def readable(self): + return self._content.readable() + class VCRConnection(object): # A reference to the cassette that's currently being patched in From 6c877a174974652c67d244d0b1c5f44493b3afc1 Mon Sep 17 00:00:00 2001 From: Rishab Malik Date: Fri, 28 Jun 2019 13:03:55 -0400 Subject: [PATCH 62/64] added skip decorator for python2 --- tests/unit/test_response.py | 63 ++++++++++++++----------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 01411ac..cde3e33 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -1,5 +1,8 @@ # coding: UTF-8 import io +import unittest + +import six from vcr.stubs import VCRHTTPResponse @@ -70,6 +73,7 @@ def test_response_headers_should_have_correct_values(): assert response.headers.get('date') == "Fri, 24 Oct 2014 18:35:37 GMT" +@unittest.skipIf(six.PY2, "Regression test for Python3 only") def test_response_parses_correctly_and_fp_attribute_error_is_not_thrown(): """ Regression test for https://github.com/kevin1024/vcrpy/issues/440 @@ -90,46 +94,25 @@ def test_response_parses_correctly_and_fp_attribute_error_is_not_thrown(): "content-type": ["text/html; charset=utf-8"], }, "body": { - "string": b"\nPMID- 19416910\nOWN - NLM\nSTAT- MEDLINE\nDA - 20090513\nDCOM- 20090622\nLR - " - b"20141209\nIS - 1091-6490 (Electronic)\nIS - 0027-8424 (Linking)\nVI - 106\nIP - " - b"19\nDP - 2009 May 12\nTI - Genetic dissection of histone deacetylase requirement in " - b"tumor cells.\nPG - 7751-5\nLID - 10.1073/pnas.0903139106 [doi]\nAB - Histone " - b"deacetylase inhibitors (HDACi) represent a new group of drugs currently\n being " - b"tested in a wide variety of clinical applications. They are especially\n effective " - b"in preclinical models of cancer where they show antiproliferative\n action in many " - b"different types of cancer cells. Recently, the first HDACi was\n approved for the " - b"treatment of cutaneous T cell lymphomas. Most HDACi currently in\n clinical " - b"development act by unspecifically interfering with the enzymatic\n activity of all " - b"class I HDACs (HDAC1, 2, 3, and 8), and it is widely believed\n that the development " - b"of isoform-specific HDACi could lead to better therapeutic\n efficacy. The " - b"contribution of the individual class I HDACs to different disease\n states, however, " - b"has so far not been fully elucidated. Here, we use a genetic\n approach to dissect " - b"the involvement of the different class I HDACs in tumor\n cells. We show that " - b"deletion of a single HDAC is not sufficient to induce cell\n death, but that HDAC1 " - b"and 2 play redundant and essential roles in tumor cell\n survival. Their deletion " - b"leads to nuclear bridging, nuclear fragmentation, and\n mitotic catastrophe, " - b"mirroring the effects of HDACi on cancer cells. These\n findings suggest that " - b"pharmacological inhibition of HDAC1 and 2 may be sufficient\n for anticancer " - b"activity, providing an experimental framework for the development \n of " - b"isoform-specific HDAC inhibitors.\nFAU - Haberland, Michael\nAU - Haberland M\nAD - " - b"Department of Molecular Biology, University of Texas Southwestern Medical Center," - b"\n 5323 Harry Hines Boulevard, Dallas, TX 75390-9148, USA.\nFAU - Johnson, " - b"Aaron\nAU - Johnson A\nFAU - Mokalled, Mayssa H\nAU - Mokalled MH\nFAU - Montgomery, " - b"Rusty L\nAU - Montgomery RL\nFAU - Olson, Eric N\nAU - Olson EN\nLA - eng\nPT - " - b"Journal Article\nPT - Research Support, N.I.H., Extramural\nPT - Research Support, " - b"Non-U.S. Gov't\nDEP - 20090429\nPL - United States\nTA - Proc Natl Acad Sci U S A\nJT " - b"- Proceedings of the National Academy of Sciences of the United States of America\nJID - " - b"7505876\nRN - 0 (Antineoplastic Agents)\nRN - 0 (Histone Deacetylase Inhibitors)\nRN - " - b"0 (Protein Isoforms)\nRN - EC 3.5.1.98 (Histone Deacetylases)\nSB - IM\nMH - " - b"Animals\nMH - Antineoplastic Agents/pharmacology\nMH - Cell Death\nMH - Cell Line, " - b"Tumor\nMH - Cell Survival\nMH - Gene Expression Regulation, Enzymologic\nMH - Gene " - b"Expression Regulation, Neoplastic\nMH - Histone Deacetylase Inhibitors\nMH - Histone " - b"Deacetylases/*genetics/*physiology\nMH - Humans\nMH - Mice\nMH - Mice, Nude\nMH - " - b"Models, Genetic\nMH - Neoplasm Transplantation\nMH - Neoplasms/metabolism\nMH - " - b"Protein Isoforms\nPMC - PMC2683118\nOID - NLM: PMC2683118\nEDAT- 2009/05/07 09:00\nMHDA- " - b"2009/06/23 09:00\nCRDT- 2009/05/07 09:00\nAID - 0903139106 [pii]\nAID - " - b"10.1073/pnas.0903139106 [doi]\nPST - ppublish\nSO - Proc Natl Acad Sci U S A. 2009 May " - b"12;106(19):7751-5. doi:\n 10.1073/pnas.0903139106. Epub 2009 Apr 29.\n " + "string": b"\nPMID- 19416910\nOWN - NLM\nSTAT- MEDLINE\nDA - 20090513\nDCOM- " + b"20090622\nLR - " + b"20141209\nIS - 1091-6490 (Electronic)\nIS - 0027-8424 (Linking)\nVI - " + b"106\nIP - " + b"19\nDP - 2009 May 12\nTI - Genetic dissection of histone deacetylase " + b"requirement in " + b"tumor cells.\nPG - 7751-5\nLID - 10.1073/pnas.0903139106 [doi]\nAB - " + b"Histone " + b"deacetylase inhibitors (HDACi) represent a new group of drugs currently\n " + b" being " + b"tested in a wide variety of clinical applications. They are especially\n " + b" effective " + b"in preclinical models of cancer where they show antiproliferative\n " + b"action in many " + b"different types of cancer cells. Recently, the first HDACi was\n " + b"approved for the " + b"treatment of cutaneous T cell lymphomas. Most HDACi currently in\n " + b"clinical " + } } vcr_response = VCRHTTPResponse(recorded_response) From d682e7b19a08adf880c0f715e61e70342da47aa8 Mon Sep 17 00:00:00 2001 From: Arthur Hamon Date: Mon, 1 Jul 2019 09:45:01 +0200 Subject: [PATCH 63/64] Fix all warnings (#449) * fix typo in pytest.mark.xskip Change xskip by skipif marker as xskip is an unknown pytest marker. * fix FileModeWarning This fix the following warning: FileModeWarning: Requests has determined the content-length for this request using the binary size of the file: however, the file has been opened in text mode (i.e. without the 'b' flag in the mode). This may lead to an incorrect content-length. In Requests 3.0, support will be removed for files in text mode. * fix waring "calling yaml.load() without Loader=... is deprecated" This fix the following warning: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. * fix collections.abc deprecation warning in python 3.7. * update Flask dependency in order to get rid of the Request.is_xhr warning This fix the following warning: DeprecationWarning: 'Request.is_xhr' is deprecated as of version 0.13 and will be removed in version 1.0. The 'X-Requested-With' header is not standard and is unreliable. You may be able to use 'accept_mimetypes' instead. --- tests/integration/test_requests.py | 10 +++++----- tests/unit/test_migration.py | 10 ++++++++-- tox.ini | 2 +- vcr/config.py | 9 ++++++--- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index f970c95..148607f 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -116,10 +116,10 @@ def test_post_chunked_binary(tmpdir, httpbin): assert req1 == req2 -@pytest.mark.xskip('sys.version_info >= (3, 6)', strict=True, raises=ConnectionError) -@pytest.mark.xskip((3, 5) < sys.version_info < (3, 6) and - platform.python_implementation() == 'CPython', - reason='Fails on CPython 3.5') +@pytest.mark.skipif('sys.version_info >= (3, 6)', strict=True, raises=ConnectionError) +@pytest.mark.skipif((3, 5) < sys.version_info < (3, 6) and + platform.python_implementation() == 'CPython', + reason='Fails on CPython 3.5') def test_post_chunked_binary_secure(tmpdir, httpbin_secure): '''Ensure that we can send chunked binary without breaking while trying to concatenate bytes with str.''' data1 = iter([b'data', b'to', b'send']) @@ -254,7 +254,7 @@ def test_nested_cassettes_with_session_created_before_nesting(httpbin_both, tmpd def test_post_file(tmpdir, httpbin_both): '''Ensure that we handle posting a file.''' url = httpbin_both + '/post' - with vcr.use_cassette(str(tmpdir.join('post_file.yaml'))) as cass, open('tox.ini') as f: + with vcr.use_cassette(str(tmpdir.join('post_file.yaml'))) as cass, open('tox.ini', 'rb') as f: original_response = requests.post(url, f).content # This also tests that we do the right thing with matching the body when they are files. diff --git a/tests/unit/test_migration.py b/tests/unit/test_migration.py index 7638910..99680a1 100644 --- a/tests/unit/test_migration.py +++ b/tests/unit/test_migration.py @@ -5,6 +5,12 @@ import yaml import vcr.migration +# Use the libYAML versions if possible +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + def test_try_migrate_with_json(tmpdir): cassette = tmpdir.join('cassette.json').strpath @@ -22,9 +28,9 @@ def test_try_migrate_with_yaml(tmpdir): shutil.copy('tests/fixtures/migration/old_cassette.yaml', cassette) assert vcr.migration.try_migrate(cassette) with open('tests/fixtures/migration/new_cassette.yaml', 'r') as f: - expected_yaml = yaml.load(f) + expected_yaml = yaml.load(f, Loader=Loader) with open(cassette, 'r') as f: - actual_yaml = yaml.load(f) + actual_yaml = yaml.load(f, Loader=Loader) assert actual_yaml == expected_yaml diff --git a/tox.ini b/tox.ini index d0542fb..eb62410 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = flake8 commands = ./runtests.sh {posargs} deps = - Flask<1 + Flask mock pytest pytest-httpbin diff --git a/vcr/config.py b/vcr/config.py index 48fb7fe..e98d580 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -1,5 +1,8 @@ import copy -import collections +try: + from collections import abc as collections_abc # only works on python 3.3+ +except ImportError: + import collections as collections_abc import functools import inspect import os @@ -175,7 +178,7 @@ class VCR(object): if decode_compressed_response: filter_functions.append(filters.decode_response) if before_record_response: - if not isinstance(before_record_response, collections.Iterable): + if not isinstance(before_record_response, collections_abc.Iterable): before_record_response = (before_record_response,) filter_functions.extend(before_record_response) @@ -241,7 +244,7 @@ class VCR(object): filter_functions.append(self._build_ignore_hosts(hosts_to_ignore)) if before_record_request: - if not isinstance(before_record_request, collections.Iterable): + if not isinstance(before_record_request, collections_abc.Iterable): before_record_request = (before_record_request,) filter_functions.extend(before_record_request) From 7c14d81ab1ba9c611ad0bd342ecdf10710b37305 Mon Sep 17 00:00:00 2001 From: Stanislav Evseev Date: Thu, 8 Nov 2018 18:41:07 +0100 Subject: [PATCH 64/64] CHANGE: return None from json if body is empty --- tests/integration/aiohttp_utils.py | 8 ++++++++ tests/integration/test_aiohttp.py | 23 +++++++++++++++++++++++ vcr/stubs/aiohttp_stubs/__init__.py | 6 +++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/integration/aiohttp_utils.py b/tests/integration/aiohttp_utils.py index c8c1b0b..851c640 100644 --- a/tests/integration/aiohttp_utils.py +++ b/tests/integration/aiohttp_utils.py @@ -30,6 +30,14 @@ def aiohttp_app(): async def hello(request): return aiohttp.web.Response(text='hello') + async def json(request): + return aiohttp.web.json_response({}) + + async def json_empty_body(request): + return aiohttp.web.json_response() + app = aiohttp.web.Application() app.router.add_get('/', hello) + app.router.add_get('/json', json) + app.router.add_get('/json/empty', json_empty_body) return app diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index a02c982..bbb0913 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -191,3 +191,26 @@ def test_aiohttp_test_client(aiohttp_client, tmpdir): response_text = loop.run_until_complete(response.text()) assert response_text == 'hello' assert cassette.play_count == 1 + + +def test_aiohttp_test_client_json(aiohttp_client, tmpdir): + loop = asyncio.get_event_loop() + app = aiohttp_app() + url = '/json/empty' + client = loop.run_until_complete(aiohttp_client(app)) + + with vcr.use_cassette(str(tmpdir.join('get.yaml'))): + response = loop.run_until_complete(client.get(url)) + + assert response.status == 200 + response_json = loop.run_until_complete(response.json()) + assert response_json is None + + with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette: + response = loop.run_until_complete(client.get(url)) + + request = cassette.requests[0] + assert request.url == str(client.make_url(url)) + response_json = loop.run_until_complete(response.json()) + assert response_json is None + assert cassette.play_count == 1 diff --git a/vcr/stubs/aiohttp_stubs/__init__.py b/vcr/stubs/aiohttp_stubs/__init__.py index 8e03907..237ddea 100644 --- a/vcr/stubs/aiohttp_stubs/__init__.py +++ b/vcr/stubs/aiohttp_stubs/__init__.py @@ -30,7 +30,11 @@ class MockClientResponse(ClientResponse): ) async def json(self, *, encoding='utf-8', loads=json.loads, **kwargs): # NOQA: E999 - return loads(self._body.decode(encoding)) + stripped = self._body.strip() + if not stripped: + return None + + return loads(stripped.decode(encoding)) async def text(self, encoding='utf-8', errors='strict'): return self._body.decode(encoding, errors=errors)