Bump version to 2.0.0

Add support to py37

.travis.yml

@@ -14,6 +14,31 @@ env:
     - TOX_SUFFIX="tornado4"
     - TOX_SUFFIX="aiohttp"
 matrix:
+  include:
+    - env: TOX_SUFFIX="flakes"
+      python: 3.7
+      dist: xenial
+      sudo: true
+    - env: TOX_SUFFIX="requests27"
+      python: 3.7
+      dist: xenial
+      sudo: true
+    - env: TOX_SUFFIX="httplib2"
+      python: 3.7
+      dist: xenial
+      sudo: true
+    - env: TOX_SUFFIX="urllib3121"
+      python: 3.7
+      dist: xenial
+      sudo: true
+    - env: TOX_SUFFIX="tornado4"
+      python: 3.7
+      dist: xenial
+      sudo: true
+    - env: TOX_SUFFIX="aiohttp"
+      python: 3.7
+      dist: xenial
+      sudo: true
   allow_failures:
     - env: TOX_SUFFIX="boto3"
     - env: TOX_SUFFIX="aiohttp"

README.rst

@@ -41,19 +41,6 @@ VCR.py will detect the absence of a cassette file and once again record
 all HTTP interactions, which will update them to correspond to the new
 API.
 
-Support
--------
-
-VCR.py works great with the following HTTP clients:
-
--  requests
--  aiohttp
--  urllib3
--  tornado
--  urllib2
--  boto3
-
-
 License
 =======
 
@@ -61,12 +48,12 @@ This library uses the MIT license. See `LICENSE.txt <LICENSE.txt>`__ for
 more details
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/vcrpy.svg
-   :target: https://pypi.python.org/pypi/vcrpy-unittest
-.. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy-unittest.svg
-   :target: https://pypi.python.org/pypi/vcrpy-unittest
-.. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.png?branch=master
+   :target: https://pypi.python.org/pypi/vcrpy
+.. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy.svg
+   :target: https://pypi.python.org/pypi/vcrpy
+.. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.svg?branch=master
    :target: http://travis-ci.org/kevin1024/vcrpy
-.. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.png?label=ready&title=waffle
+.. |Waffle Ready| image:: https://badge.waffle.io/kevin1024/vcrpy.svg?label=ready&title=waffle
    :target: https://waffle.io/kevin1024/vcrpy
 .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
    :alt: Join the chat at https://gitter.im/kevin1024/vcrpy

docs/changelog.rst

@@ -1,5 +1,13 @@
 Changelog
 ---------
+-  2.0.0 - Support python 3.7 (fix httplib2 and urllib2, thanks @felixonmars)
+   [#356] Fixes `before_record_response` so the original response isn't changed (thanks @kgraves)
+   Fix requests stub when using proxy (thanks @samuelfekete @daneoshiga)
+   (only for aiohttp stub) Drop support to python 3.4 asyncio.coroutine (aiohttp doesn't support python it anymore)
+   Fix aiohttp stub to work with aiohttp client (thanks @stj)
+   Fix aiohttp stub to accept content type passed
+   Improve docs (thanks @adamchainz)
+-  1.13.0 - Fix support to latest aiohttp version (3.3.2). Fix content-type bug in aiohttp stub. Save URL with query params properly when using aiohttp.
 -  1.12.0 - Fix support to latest aiohttp version (3.2.1), Adapted setup to PEP508, Support binary responses on aiohttp, Dropped support for EOL python versions (2.6 and 3.3)
 -  1.11.1 Fix compatibility with newest requests and urllib3 releases
 -  1.11.0 Allow injection of persistence methods + bugfixes (thanks @j-funk and @IvanMalison),

docs/installation.rst

@@ -12,15 +12,17 @@ Compatibility
 VCR.py supports Python 2.7 and 3.4+, and
 `pypy <http://pypy.org>`__.
 
-The following http libraries are supported:
+The following HTTP libraries are supported:
 
--  urllib2
--  urllib3
--  http.client (python3)
--  requests (both 1.x and 2.x versions)
--  httplib2
--  boto
--  Tornado's AsyncHTTPClient
+-  ``aiohttp``
+-  ``boto``
+-  ``boto3``
+-  ``http.client``
+-  ``httplib2``
+-  ``requests`` (both 1.x and 2.x versions)
+-  ``tornado.httpclient``
+-  ``urllib2``
+-  ``urllib3``
 
 Speed
 -----

setup.py

@@ -37,7 +37,7 @@ if sys.version_info[0] == 2:
 
 setup(
     name='vcrpy',
-    version='1.12.0',
+    version='2.0.0',
     description=(
         "Automatically mock your HTTP interactions to simplify and "
         "speed up testing"
@@ -62,6 +62,7 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Software Development :: Testing',

tests/integration/aiohttp_utils.py

@@ -2,22 +2,32 @@
 import asyncio
 
 import aiohttp
+from aiohttp.test_utils import TestClient
 
 
-@asyncio.coroutine
-def aiohttp_request(loop, method, url, output='text', encoding='utf-8', **kwargs):
+async def aiohttp_request(loop, method, url, output='text', encoding='utf-8', content_type=None, **kwargs):
     session = aiohttp.ClientSession(loop=loop)
     response_ctx = session.request(method, url, **kwargs)
 
-    response = yield from response_ctx.__aenter__()
+    response = await response_ctx.__aenter__()
     if output == 'text':
-        content = yield from response.text()
+        content = await response.text()
     elif output == 'json':
-        content = yield from response.json(encoding=encoding)
+        content_type = content_type or 'application/json'
+        content = await response.json(encoding=encoding, content_type=content_type)
     elif output == 'raw':
-        content = yield from response.read()
+        content = await response.read()
 
     response_ctx._resp.close()
-    yield from session.close()
+    await session.close()
 
     return response, content
+
+
+def aiohttp_app():
+    async def hello(request):
+        return aiohttp.web.Response(text='hello')
+
+    app = aiohttp.web.Application()
+    app.router.add_get('/', hello)
+    return app

tests/integration/test_aiohttp.py

@@ -5,7 +5,7 @@ asyncio = pytest.importorskip("asyncio")
 aiohttp = pytest.importorskip("aiohttp")
 
 import vcr  # noqa: E402
-from .aiohttp_utils import aiohttp_request  # noqa: E402
+from .aiohttp_utils import aiohttp_app, aiohttp_request  # noqa: E402
 
 
 def run_in_loop(fn):
@@ -137,3 +137,43 @@ def test_params_same_url_distinct_params(tmpdir, scheme):
         response, cassette_response_text = get(url, output='text', params=other_params)
         assert 'No match for the request' in cassette_response_text
         assert response.status == 599
+
+
+def test_params_on_url(tmpdir, scheme):
+    url = scheme + '://httpbin.org/get?a=1&b=foo'
+    headers = {'Content-Type': 'application/json'}
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette:
+        _, response_json = get(url, output='json', headers=headers)
+        request = cassette.requests[0]
+        assert request.url == url
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette:
+        _, cassette_response_json = get(url, output='json', headers=headers)
+        request = cassette.requests[0]
+        assert request.url == url
+        assert cassette_response_json == response_json
+        assert cassette.play_count == 1
+
+
+def test_aiohttp_test_client(aiohttp_client, tmpdir):
+    loop = asyncio.get_event_loop()
+    app = aiohttp_app()
+    url = '/'
+    client = loop.run_until_complete(aiohttp_client(app))
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))):
+        response = loop.run_until_complete(client.get(url))
+
+    assert response.status == 200
+    response_text = loop.run_until_complete(response.text())
+    assert response_text == 'hello'
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette:
+        response = loop.run_until_complete(client.get(url))
+
+    request = cassette.requests[0]
+    assert request.url == str(client.make_url(url))
+    response_text = loop.run_until_complete(response.text())
+    assert response_text == 'hello'
+    assert cassette.play_count == 1

tests/integration/test_httplib2.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 '''Integration tests with httplib2'''
 
-# External imports
+import sys
+
 from six.moves.urllib_parse import urlencode
 import pytest
 import pytest_httpbin.certs
 
-# Internal imports
 import vcr
 
 from assertions import assert_cassette_has_one_response
@@ -19,7 +19,12 @@ def http():
     Returns an httplib2 HTTP instance
     with the certificate replaced by the httpbin one.
     """
-    return httplib2.Http(ca_certs=pytest_httpbin.certs.where())
+    kwargs = {
+        'ca_certs': pytest_httpbin.certs.where()
+    }
+    if sys.version_info[:2] == (3, 7):
+        kwargs['disable_ssl_certificate_validation'] = True
+    return httplib2.Http(**kwargs)
 
 
 def test_response_code(tmpdir, httpbin_both):

tests/integration/test_proxy.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+'''Test using a proxy.'''
+
+# External imports
+import multiprocessing
+import pytest
+
+from six.moves import socketserver, SimpleHTTPServer
+from six.moves.urllib.request import urlopen
+
+# Internal imports
+import vcr
+
+# Conditional imports
+requests = pytest.importorskip("requests")
+
+
+class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    '''
+    Simple proxy server.
+
+    (Inspired by: http://effbot.org/librarybook/simplehttpserver.htm).
+    '''
+    def do_GET(self):
+        upstream_response = urlopen(self.path)
+        try:
+            status = upstream_response.status
+            headers = upstream_response.headers.items()
+        except AttributeError:
+            # In Python 2 the response is an addinfourl instance.
+            status = upstream_response.code
+            headers = upstream_response.info().items()
+        self.send_response(status, upstream_response.msg)
+        for header in headers:
+            self.send_header(*header)
+        self.end_headers()
+        self.copyfile(upstream_response, self.wfile)
+
+
+@pytest.yield_fixture(scope='session')
+def proxy_server():
+    httpd = socketserver.ThreadingTCPServer(('', 0), Proxy)
+    proxy_process = multiprocessing.Process(
+        target=httpd.serve_forever,
+    )
+    proxy_process.start()
+    yield 'http://{0}:{1}'.format(*httpd.server_address)
+    proxy_process.terminate()
+
+
+def test_use_proxy(tmpdir, httpbin, proxy_server):
+    '''Ensure that it works with a proxy.'''
+    with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))):
+        response = requests.get(httpbin.url, proxies={'http': proxy_server})
+
+    with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette:
+        cassette_response = requests.get(httpbin.url, proxies={'http': proxy_server})
+
+    assert cassette_response.headers == response.headers
+    assert cassette.play_count == 1

tests/integration/test_stubs.py

@@ -1,5 +1,6 @@
 import vcr
 import zlib
+import json
 import six.moves.http_client as httplib
 
 from assertions import assert_is_json
@@ -83,3 +84,50 @@ def test_original_decoded_response_is_not_modified(tmpdir, httpbin):
 
         assert 'content-encoding' not in inside.headers
         assert_is_json(inside.read())
+
+
+def _make_before_record_response(fields, replacement='[REDACTED]'):
+    def before_record_response(response):
+        string_body = response['body']['string'].decode('utf8')
+        body = json.loads(string_body)
+
+        for field in fields:
+            if field in body:
+                body[field] = replacement
+
+        response['body']['string'] = json.dumps(body).encode()
+        return response
+    return before_record_response
+
+
+def test_original_response_is_not_modified_by_before_filter(tmpdir, httpbin):
+    testfile = str(tmpdir.join('sensitive_data_scrubbed_response.yml'))
+    host, port = httpbin.host, httpbin.port
+    field_to_scrub = 'url'
+    replacement = '[YOU_CANT_HAVE_THE_MANGO]'
+
+    conn = httplib.HTTPConnection(host, port)
+    conn.request('GET', '/get')
+    outside = conn.getresponse()
+
+    callback = _make_before_record_response([field_to_scrub], replacement)
+    with vcr.use_cassette(testfile, before_record_response=callback):
+        conn = httplib.HTTPConnection(host, port)
+        conn.request('GET', '/get')
+        inside = conn.getresponse()
+
+        # The scrubbed field should be the same, because no cassette existed.
+        # Furthermore, the responses should be identical.
+        inside_body = json.loads(inside.read().decode('utf-8'))
+        outside_body = json.loads(outside.read().decode('utf-8'))
+        assert not inside_body[field_to_scrub] == replacement
+        assert inside_body[field_to_scrub] == outside_body[field_to_scrub]
+
+    # Ensure that when a cassette exists, the scrubbed response is returned.
+    with vcr.use_cassette(testfile, before_record_response=callback):
+        conn = httplib.HTTPConnection(host, port)
+        conn.request('GET', '/get')
+        inside = conn.getresponse()
+
+        inside_body = json.loads(inside.read().decode('utf-8'))
+        assert inside_body[field_to_scrub] == replacement

tests/integration/test_urllib2.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 '''Integration tests with urllib2'''
 
+import ssl
 from six.moves.urllib.request import urlopen
 from six.moves.urllib_parse import urlencode
 import pytest_httpbin.certs
@@ -12,7 +13,9 @@ from assertions import assert_cassette_has_one_response
 
 
 def urlopen_with_cafile(*args, **kwargs):
-    kwargs['cafile'] = pytest_httpbin.certs.where()
+    context = ssl.create_default_context(cafile=pytest_httpbin.certs.where())
+    context.check_hostname = False
+    kwargs['context'] = context
     try:
         return urlopen(*args, **kwargs)
     except TypeError:

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,py36,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp}
+envlist = {py27,py35,py36,py37,pypy}-{flakes,requests27,httplib2,urllib3121,tornado4,boto3,aiohttp}
 
 [testenv:flakes]
 skipsdist = True
@@ -27,6 +27,7 @@ deps =
     boto3: boto3
     aiohttp: aiohttp
     aiohttp: pytest-asyncio
+    aiohttp: pytest-aiohttp
 
 [flake8]
 max_line_length = 110

vcr/_handle_coroutine.py

@@ -1,7 +1,3 @@
-import asyncio
-
-
-@asyncio.coroutine
-def handle_coroutine(vcr, fn):
+async def handle_coroutine(vcr, fn):  # noqa: E999
     with vcr as cassette:
-        return (yield from fn(cassette))  # noqa: E999
+        return (await fn(cassette))  # noqa: E999

vcr/cassette.py

@@ -1,4 +1,5 @@
 import collections
+import copy
 import sys
 import inspect
 import logging
@@ -135,7 +136,10 @@ class CassetteContextDecorator(object):
                 except Exception:
                     to_yield = coroutine.throw(*sys.exc_info())
                 else:
-                    to_yield = coroutine.send(to_send)
+                    try:
+                        to_yield = coroutine.send(to_send)
+                    except StopIteration:
+                        break
 
     def _handle_function(self, fn):
         with self as cassette:
@@ -222,6 +226,9 @@ class Cassette(object):
         request = self._before_record_request(request)
         if not request:
             return
+        # Deepcopy is here because mutation of `response` will corrupt the
+        # real response.
+        response = copy.deepcopy(response)
         response = self._before_record_response(response)
         if response is None:
             return

vcr/stubs/__init__.py

@@ -18,7 +18,7 @@ log = logging.getLogger(__name__)
 class VCRFakeSocket(object):
     """
     A socket that doesn't do anything!
-    Used when playing back casssettes, when there
+    Used when playing back cassettes, when there
     is no actual open socket.
     """
 
@@ -136,7 +136,10 @@ class VCRConnection(object):
 
     def _uri(self, url):
         """Returns request absolute URI"""
-        uri = "{}://{}{}{}".format(
+        if url and not url.startswith('/'):
+            # Then this must be a proxy request.
+            return url
+        uri = "{0}://{1}{2}{3}".format(
             self._protocol,
             self.real_connection.host,
             self._port_postfix(),
@@ -168,6 +171,8 @@ class VCRConnection(object):
         # allows me to compare the entire length of the response to see if it
         # exists in the cassette.
 
+        self._sock = VCRFakeSocket()
+
     def putrequest(self, method, url, *args, **kwargs):
         """
         httplib gives you more than one way to do it.  This is a way
@@ -291,11 +296,13 @@ class VCRConnection(object):
         with force_reset():
             return self.real_connection.connect(*args, **kwargs)
 
+        self._sock = VCRFakeSocket()
+
     @property
     def sock(self):
         if self.real_connection.sock:
             return self.real_connection.sock
-        return VCRFakeSocket()
+        return self._sock
 
     @sock.setter
     def sock(self, value):
@@ -313,6 +320,8 @@ class VCRConnection(object):
         with force_reset():
             self.real_connection = self._baseclass(*args, **kwargs)
 
+        self._sock = None
+
     def __setattr__(self, name, value):
         """
         We need to define this because any attributes that are set on the

vcr/stubs/aiohttp_stubs/__init__.py

@@ -20,43 +20,38 @@ class MockClientResponse(ClientResponse):
             continue100=None,
             timer=None,
             request_info=None,
-            auto_decompress=None,
             traces=None,
             loop=asyncio.get_event_loop(),
             session=None,
         )
 
-    # TODO: get encoding from header
-    @asyncio.coroutine
-    def json(self, *, encoding='utf-8', loads=json.loads):  # NOQA: E999
-        return loads(self.content.decode(encoding))
+    async def json(self, *, encoding='utf-8', loads=json.loads, **kwargs):  # NOQA: E999
+        return loads(self._body.decode(encoding))
 
-    @asyncio.coroutine
-    def text(self, encoding='utf-8'):
-        return self.content.decode(encoding)
+    async def text(self, encoding='utf-8'):
+        return self._body.decode(encoding)
 
-    @asyncio.coroutine
-    def read(self):
-        return self.content
+    async def read(self):
+        return self._body
 
-    @asyncio.coroutine
-    def release(self):
+    async def release(self):
         pass
 
 
 def vcr_request(cassette, real_request):
     @functools.wraps(real_request)
-    @asyncio.coroutine
-    def new_request(self, method, url, **kwargs):
+    async def new_request(self, method, url, **kwargs):
         headers = kwargs.get('headers')
         headers = self._prepare_headers(headers)
         data = kwargs.get('data')
         params = kwargs.get('params')
+
+        request_url = URL(url)
         if params:
             for k, v in params.items():
                 params[k] = str(v)
+            request_url = URL(url).with_query(params)
 
-        request_url = URL(url).with_query(params)
         vcr_request = Request(method, str(request_url), data, headers)
 
         if cassette.can_play_response_for(vcr_request):
@@ -64,9 +59,9 @@ def vcr_request(cassette, real_request):
 
             response = MockClientResponse(method, URL(vcr_response.get('url')))
             response.status = vcr_response['status']['code']
-            response.content = vcr_response['body']['string']
+            response._body = vcr_response['body']['string']
             response.reason = vcr_response['status']['message']
-            response.headers = vcr_response['headers']
+            response._headers = vcr_response['headers']
 
             response.close()
             return response
@@ -77,11 +72,11 @@ def vcr_request(cassette, real_request):
             msg = ("No match for the request {!r} was found. Can't overwrite "
                    "existing cassette {!r} in your current record mode {!r}.")
             msg = msg.format(vcr_request, cassette._path, cassette.record_mode)
-            response.content = msg.encode()
+            response._body = msg.encode()
             response.close()
             return response
 
-        response = yield from real_request(self, method, url, **kwargs)  # NOQA: E999
+        response = await real_request(self, method, url, **kwargs)  # NOQA: E999
 
         vcr_response = {
             'status': {
@@ -89,7 +84,7 @@ def vcr_request(cassette, real_request):
                 'message': response.reason,
             },
             'headers': dict(response.headers),
-            'body': {'string': (yield from response.read())},  # NOQA: E999
+            'body': {'string': (await response.read())},  # NOQA: E999
             'url': response.url,
         }
         cassette.append(vcr_request, vcr_response)

vcr/stubs/httplib2_stubs.py

@@ -40,6 +40,7 @@ class VCRHTTPSConnectionWithTimeout(VCRHTTPSConnection,
             'timeout',
             'source_address',
             'ca_certs',
+            'disable_ssl_certificate_validation',
         }
         unknown_keys = set(kwargs.keys()) - safe_keys
         safe_kwargs = kwargs.copy()