bump version to v4.3.1

.. because it turned out broken for Python <3.10.

.github/workflows/main.yml

@@ -16,11 +16,6 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8"]
 
     steps:
-      - name: Install libgnutls28-dev
-        run: |
-          sudo apt update -q
-          sudo apt install -q -y libgnutls28-dev libcurl4-gnutls-dev
-
       - uses: actions/checkout@v3.5.2
 
       - name: Set up Python ${{ matrix.python-version }}

docs/changelog.rst

@@ -7,6 +7,18 @@ For a full list of triaged issues, bugs and PRs and what release they are target
 
 All help in providing PRs to close out bug issues is appreciated. Even if that is providing a repo that fully replicates issues. We have very generous contributors that have added these to bug issues which meant another contributor picked up the bug and closed it out.
 
+-  4.3.1
+    - Support urllib3 v1 and v2.  NOTE: there is an issue running urllib3 v2 on
+      Python older than 3.10, so this is currently blocked in the requirements.
+      Hopefully we can resolve this situation in the future.  Thanks to @shifqu,
+      hartwork, jairhenrique, pquentin, and vEpiphyte for your work on this.
+-  4.3.0
+    - Add support for Python 3.11 (Thanks @evgeni)
+    - Drop support for botocore <1.11.0 and requests <2.16.2 (thanks @hartwork)
+    - Bugfix: decode_compressed_response raises exception on empty responses. Thanks @CharString
+    - Don't save requests from decorated tests if decorated test fails (thanks @dan-passaro)
+    - Fix not calling all the exit stack when record_on_exception is False (thanks @Terseus)
+    - Various CI, documentation, testing, and formatting improvements (Thanks @jairhenrique, @dan-passaro, @hartwork, and Terseus)
 -  4.2.1
     - Fix a bug where the first request in a redirect chain was not being recorded with aiohttp
     - Various typos and small fixes, thanks @jairhenrique, @timgates42

setup.py

@@ -47,6 +47,32 @@ install_requires = [
     "wrapt",
     "six>=1.5",
     "yarl",
+    # Support for urllib3 >=2 needs Python >=3.10
+    # so we need to block urllib3 >=2 for Python <3.10 for now.
+    # Note that vcrpy would work fine without any urllib3 around,
+    # so this block and the dependency can be dropped at some point
+    # in the future. For more Details:
+    # https://github.com/kevin1024/vcrpy/pull/699#issuecomment-1551439663
+    "urllib3 <2; python_version <'3.10'",
+]
+
+tests_require = [
+    "aiohttp",
+    "boto3",
+    "httplib2",
+    "httpx",
+    "pytest",
+    "pytest-aiohttp",
+    "pytest-httpbin",
+    "requests",
+    "tornado",
+    # Needed to un-break httpbin 0.7.0. For httpbin >=0.7.1 and after,
+    # this pin and the dependency itself can be removed, provided
+    # that the related bug in httpbin has been fixed:
+    # https://github.com/kevin1024/vcrpy/issues/645#issuecomment-1562489489
+    # https://github.com/postmanlabs/httpbin/issues/673
+    # https://github.com/postmanlabs/httpbin/pull/674
+    "Werkzeug==2.0.3",
 ]
 
 setup(
@@ -62,7 +88,7 @@ setup(
     python_requires=">=3.7",
     install_requires=install_requires,
     license="MIT",
-    tests_require=["pytest", "mock", "pytest-httpbin"],
+    tests_require=tests_require,
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",

tests/integration/conftest.py

@@ -1,3 +1,6 @@
+import os
+import ssl
+
 import pytest
 
 
@@ -15,3 +18,15 @@ def mockbin(scheme):
 @pytest.fixture
 def mockbin_request_url(mockbin):
     return mockbin + "/request"
+
+
+@pytest.fixture
+def httpbin_ssl_context():
+    ssl_ca_location = os.environ["REQUESTS_CA_BUNDLE"]
+    ssl_cert_location = os.environ["REQUESTS_CA_BUNDLE"].replace("cacert.pem", "cert.pem")
+    ssl_key_location = os.environ["REQUESTS_CA_BUNDLE"].replace("cacert.pem", "key.pem")
+
+    ssl_context = ssl.create_default_context(cafile=ssl_ca_location)
+    ssl_context.load_cert_chain(ssl_cert_location, ssl_key_location)
+
+    return ssl_context

tests/integration/test_aiohttp.py

@@ -320,31 +320,35 @@ def test_double_requests(tmpdir, mockbin_request_url):
         assert cassette.play_count == 2
 
 
-def test_cookies(scheme, tmpdir):
+def test_cookies(httpbin_both, httpbin_ssl_context, tmpdir):
     async def run(loop):
-        cookies_url = scheme + (
-            "://httpbin.org/response-headers?"
+        cookies_url = httpbin_both.url + (
+            "/response-headers?"
             "set-cookie=" + urllib.parse.quote("cookie_1=val_1; Path=/") + "&"
             "Set-Cookie=" + urllib.parse.quote("Cookie_2=Val_2; Path=/")
         )
-        home_url = scheme + "://httpbin.org/"
+        home_url = httpbin_both.url + "/"
         tmp = str(tmpdir.join("cookies.yaml"))
         req_cookies = {"Cookie_3": "Val_3"}
         req_headers = {"Cookie": "Cookie_4=Val_4"}
 
         # ------------------------- Record -------------------------- #
         with vcr.use_cassette(tmp) as cassette:
-            async with aiohttp.ClientSession(loop=loop) as session:
-                cookies_resp = await session.get(cookies_url)
-                home_resp = await session.get(home_url, cookies=req_cookies, headers=req_headers)
+            async with aiohttp.ClientSession(loop=loop, cookie_jar=aiohttp.CookieJar(unsafe=True)) as session:
+                cookies_resp = await session.get(cookies_url, ssl=httpbin_ssl_context)
+                home_resp = await session.get(
+                    home_url, cookies=req_cookies, headers=req_headers, ssl=httpbin_ssl_context
+                )
                 assert cassette.play_count == 0
         assert_responses(cookies_resp, home_resp)
 
         # -------------------------- Play --------------------------- #
         with vcr.use_cassette(tmp, record_mode=vcr.mode.NONE) as cassette:
-            async with aiohttp.ClientSession(loop=loop) as session:
-                cookies_resp = await session.get(cookies_url)
-                home_resp = await session.get(home_url, cookies=req_cookies, headers=req_headers)
+            async with aiohttp.ClientSession(loop=loop, cookie_jar=aiohttp.CookieJar(unsafe=True)) as session:
+                cookies_resp = await session.get(cookies_url, ssl=httpbin_ssl_context)
+                home_resp = await session.get(
+                    home_url, cookies=req_cookies, headers=req_headers, ssl=httpbin_ssl_context
+                )
                 assert cassette.play_count == 2
         assert_responses(cookies_resp, home_resp)
 
@@ -360,16 +364,16 @@ def test_cookies(scheme, tmpdir):
     run_in_loop(run)
 
 
-def test_cookies_redirect(scheme, tmpdir):
+def test_cookies_redirect(httpbin_both, httpbin_ssl_context, tmpdir):
     async def run(loop):
         # Sets cookie as provided by the query string and redirects
-        cookies_url = scheme + "://httpbin.org/cookies/set?Cookie_1=Val_1"
+        cookies_url = httpbin_both.url + "/cookies/set?Cookie_1=Val_1"
         tmp = str(tmpdir.join("cookies.yaml"))
 
         # ------------------------- Record -------------------------- #
         with vcr.use_cassette(tmp) as cassette:
-            async with aiohttp.ClientSession(loop=loop) as session:
-                cookies_resp = await session.get(cookies_url)
+            async with aiohttp.ClientSession(loop=loop, cookie_jar=aiohttp.CookieJar(unsafe=True)) as session:
+                cookies_resp = await session.get(cookies_url, ssl=httpbin_ssl_context)
                 assert not cookies_resp.cookies
                 cookies = session.cookie_jar.filter_cookies(cookies_url)
                 assert cookies["Cookie_1"].value == "Val_1"
@@ -378,8 +382,8 @@ def test_cookies_redirect(scheme, tmpdir):
 
         # -------------------------- Play --------------------------- #
         with vcr.use_cassette(tmp, record_mode=vcr.mode.NONE) as cassette:
-            async with aiohttp.ClientSession(loop=loop) as session:
-                cookies_resp = await session.get(cookies_url)
+            async with aiohttp.ClientSession(loop=loop, cookie_jar=aiohttp.CookieJar(unsafe=True)) as session:
+                cookies_resp = await session.get(cookies_url, ssl=httpbin_ssl_context)
                 assert not cookies_resp.cookies
                 cookies = session.cookie_jar.filter_cookies(cookies_url)
                 assert cookies["Cookie_1"].value == "Val_1"
@@ -391,8 +395,8 @@ def test_cookies_redirect(scheme, tmpdir):
             cassette.responses[0]["headers"]["set-cookie"] = [
                 "Cookie_1=Val_1; Expires=Wed, 21 Oct 2015 07:28:00 GMT"
             ]
-            async with aiohttp.ClientSession(loop=loop) as session:
-                cookies_resp = await session.get(cookies_url)
+            async with aiohttp.ClientSession(loop=loop, cookie_jar=aiohttp.CookieJar(unsafe=True)) as session:
+                cookies_resp = await session.get(cookies_url, ssl=httpbin_ssl_context)
                 assert not cookies_resp.cookies
                 cookies = session.cookie_jar.filter_cookies(cookies_url)
                 assert cookies["Cookie_1"].value == "Val_1"

tests/integration/test_urllib3.py

@@ -8,6 +8,7 @@ from assertions import assert_cassette_empty, assert_is_json
 
 import vcr
 from vcr.patch import force_reset
+from vcr.stubs.compat import get_headers
 
 urllib3 = pytest.importorskip("urllib3")
 
@@ -41,7 +42,8 @@ def test_headers(tmpdir, httpbin_both, verify_pool_mgr):
         headers = verify_pool_mgr.request("GET", url).headers
 
     with vcr.use_cassette(str(tmpdir.join("headers.yaml"))):
-        assert headers == verify_pool_mgr.request("GET", url).headers
+        new_headers = verify_pool_mgr.request("GET", url).headers
+        assert sorted(get_headers(headers)) == sorted(get_headers(new_headers))
 
 
 def test_body(tmpdir, httpbin_both, verify_pool_mgr):
@@ -145,18 +147,18 @@ def test_https_with_cert_validation_disabled(tmpdir, httpbin_secure, pool_mgr):
 
 
 def test_urllib3_force_reset():
-    cpool = urllib3.connectionpool
-    http_original = cpool.HTTPConnection
-    https_original = cpool.HTTPSConnection
-    verified_https_original = cpool.VerifiedHTTPSConnection
+    conn = urllib3.connection
+    http_original = conn.HTTPConnection
+    https_original = conn.HTTPSConnection
+    verified_https_original = conn.VerifiedHTTPSConnection
     with vcr.use_cassette(path="test"):
-        first_cassette_HTTPConnection = cpool.HTTPConnection
-        first_cassette_HTTPSConnection = cpool.HTTPSConnection
-        first_cassette_VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection
+        first_cassette_HTTPConnection = conn.HTTPConnection
+        first_cassette_HTTPSConnection = conn.HTTPSConnection
+        first_cassette_VerifiedHTTPSConnection = conn.VerifiedHTTPSConnection
         with force_reset():
-            assert cpool.HTTPConnection is http_original
-            assert cpool.HTTPSConnection is https_original
-            assert cpool.VerifiedHTTPSConnection is verified_https_original
-        assert cpool.HTTPConnection is first_cassette_HTTPConnection
-        assert cpool.HTTPSConnection is first_cassette_HTTPSConnection
-        assert cpool.VerifiedHTTPSConnection is first_cassette_VerifiedHTTPSConnection
+            assert conn.HTTPConnection is http_original
+            assert conn.HTTPSConnection is https_original
+            assert conn.VerifiedHTTPSConnection is verified_https_original
+        assert conn.HTTPConnection is first_cassette_HTTPConnection
+        assert conn.HTTPSConnection is first_cassette_HTTPSConnection
+        assert conn.VerifiedHTTPSConnection is first_cassette_VerifiedHTTPSConnection

tests/integration/test_wild.py

@@ -64,9 +64,10 @@ def test_cookies(tmpdir, httpbin):
     with vcr.use_cassette(testfile):
         s = requests.Session()
         s.get(httpbin.url + "/cookies/set?k1=v1&k2=v2")
+        assert s.cookies.keys() == ["k1", "k2"]
 
         r2 = s.get(httpbin.url + "/cookies")
-        assert len(r2.json()["cookies"]) == 2
+        assert sorted(r2.json()["cookies"].keys()) == ["k1", "k2"]
 
 
 def test_amazon_doctype(tmpdir):

tests/unit/test_vcr.py

@@ -41,7 +41,7 @@ def test_vcr_use_cassette():
 
 
 def test_vcr_before_record_request_params():
-    base_path = "http://httpbin.org/"
+    base_path = "http://whatever.test/"
 
     def before_record_cb(request):
         if request.path != "/get":

tox.ini

@@ -3,8 +3,9 @@ skip_missing_interpreters=true
 envlist =
   cov-clean,
   lint,
-  {py37,py38,py39,py310,py311}-{requests,httplib2,urllib3,tornado4,boto3,aiohttp,httpx},
-  {pypy3}-{requests,httplib2,urllib3,tornado4,boto3},
+  {py37,py38,py39,py310,py311}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3,aiohttp,httpx},
+  {py310,py311}-{requests-urllib3-2,urllib3-2},
+  {pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3},
   {py310}-httpx019,
   cov-report
 
@@ -85,11 +86,10 @@ deps =
     PyYAML
     ipaddress
     requests: requests>=2.22.0
-    requests: urllib3<2
     httplib2: httplib2
-    urllib3: urllib3<2
+    urllib3-1: urllib3<2
+    urllib3-2: urllib3<3
     boto3: boto3
-    boto3: urllib3
     aiohttp: aiohttp
     aiohttp: pytest-asyncio
     aiohttp: pytest-aiohttp
@@ -101,8 +101,8 @@ deps =
     httpx019: httpx==0.19
     {py37,py38,py39,py310}-{httpx}: pytest-asyncio
 depends =
-  lint,{py37,py38,py39,py310,py311,pypy3}-{requests,httplib2,urllib3,tornado4,boto3},{py37,py38,py39,py310,py311}-{aiohttp},{py37,py38,py39,py310,py311}-{httpx}: cov-clean
-  cov-report: lint,{py37,py38,py39,py310,py311,pypy3}-{requests,httplib2,urllib3,tornado4,boto3},{py37,py38,py39,py310,py311}-{aiohttp}
+  lint,{py37,py38,py39,py310,py311,pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3},{py310,py311}-{requests-urllib3-2,urllib3-2},{py37,py38,py39,py310,py311}-{aiohttp},{py37,py38,py39,py310,py311}-{httpx}: cov-clean
+  cov-report: lint,{py37,py38,py39,py310,py311,pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3},{py310,py311}-{requests-urllib3-2,urllib3-2},{py37,py38,py39,py310,py311}-{aiohttp}
 passenv =
     AWS_ACCESS_KEY_ID
     AWS_DEFAULT_REGION

vcr/__init__.py

@@ -4,7 +4,7 @@ from logging import NullHandler
 from .config import VCR
 from .record_mode import RecordMode as mode  # noqa import is not used in this file
 
-__version__ = "4.2.1"
+__version__ = "4.3.1"
 
 logging.getLogger(__name__).addHandler(NullHandler())
 

vcr/patch.py

@@ -32,15 +32,17 @@ else:
     _cpoolBoto3HTTPSConnection = AWSHTTPSConnection
 
 cpool = None
+conn = None
 # Try to save the original types for urllib3
 try:
+    import urllib3.connection as conn
     import urllib3.connectionpool as cpool
 except ImportError:  # pragma: no cover
     pass
 else:
-    _VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection
-    _cpoolHTTPConnection = cpool.HTTPConnection
-    _cpoolHTTPSConnection = cpool.HTTPSConnection
+    _VerifiedHTTPSConnection = conn.VerifiedHTTPSConnection
+    _connHTTPConnection = conn.HTTPConnection
+    _connHTTPSConnection = conn.HTTPSConnection
 
 # Try to save the original types for requests
 try:
@@ -198,7 +200,7 @@ class CassettePatcherBuilder:
             from .stubs import requests_stubs
         except ImportError:  # pragma: no cover
             return ()
-        return self._urllib3_patchers(cpool, requests_stubs)
+        return self._urllib3_patchers(cpool, conn, requests_stubs)
 
     @_build_patchers_from_mock_triples_decorator
     def _boto3(self):
@@ -248,12 +250,13 @@ class CassettePatcherBuilder:
 
     def _urllib3(self):
         try:
+            import urllib3.connection as conn
             import urllib3.connectionpool as cpool
         except ImportError:  # pragma: no cover
             return ()
         from .stubs import urllib3_stubs
 
-        return self._urllib3_patchers(cpool, urllib3_stubs)
+        return self._urllib3_patchers(cpool, conn, urllib3_stubs)
 
     @_build_patchers_from_mock_triples_decorator
     def _httplib2(self):
@@ -330,7 +333,7 @@ class CassettePatcherBuilder:
             new_sync_client_send = sync_vcr_send(self._cassette, _HttpxSyncClient_send)
             yield httpx.Client, "send", new_sync_client_send
 
-    def _urllib3_patchers(self, cpool, stubs):
+    def _urllib3_patchers(self, cpool, conn, stubs):
         http_connection_remover = ConnectionRemover(
             self._get_cassette_subclass(stubs.VCRRequestsHTTPConnection)
         )
@@ -338,9 +341,9 @@ class CassettePatcherBuilder:
             self._get_cassette_subclass(stubs.VCRRequestsHTTPSConnection)
         )
         mock_triples = (
-            (cpool, "VerifiedHTTPSConnection", stubs.VCRRequestsHTTPSConnection),
-            (cpool, "HTTPConnection", stubs.VCRRequestsHTTPConnection),
-            (cpool, "HTTPSConnection", stubs.VCRRequestsHTTPSConnection),
+            (conn, "VerifiedHTTPSConnection", stubs.VCRRequestsHTTPSConnection),
+            (conn, "HTTPConnection", stubs.VCRRequestsHTTPConnection),
+            (conn, "HTTPSConnection", stubs.VCRRequestsHTTPSConnection),
             (cpool, "is_connection_dropped", mock.Mock(return_value=False)),  # Needed on Windows only
             (cpool.HTTPConnectionPool, "ConnectionCls", stubs.VCRRequestsHTTPConnection),
             (cpool.HTTPSConnectionPool, "ConnectionCls", stubs.VCRRequestsHTTPSConnection),
@@ -410,16 +413,17 @@ def reset_patchers():
     yield mock.patch.object(httplib, "HTTPSConnection", _HTTPSConnection)
 
     try:
+        import urllib3.connection as conn
         import urllib3.connectionpool as cpool
     except ImportError:  # pragma: no cover
         pass
     else:
-        yield mock.patch.object(cpool, "VerifiedHTTPSConnection", _VerifiedHTTPSConnection)
-        yield mock.patch.object(cpool, "HTTPConnection", _cpoolHTTPConnection)
-        yield mock.patch.object(cpool, "HTTPSConnection", _cpoolHTTPSConnection)
+        yield mock.patch.object(conn, "VerifiedHTTPSConnection", _VerifiedHTTPSConnection)
+        yield mock.patch.object(conn, "HTTPConnection", _connHTTPConnection)
+        yield mock.patch.object(conn, "HTTPSConnection", _connHTTPSConnection)
         if hasattr(cpool.HTTPConnectionPool, "ConnectionCls"):
-            yield mock.patch.object(cpool.HTTPConnectionPool, "ConnectionCls", _cpoolHTTPConnection)
-            yield mock.patch.object(cpool.HTTPSConnectionPool, "ConnectionCls", _cpoolHTTPSConnection)
+            yield mock.patch.object(cpool.HTTPConnectionPool, "ConnectionCls", _connHTTPConnection)
+            yield mock.patch.object(cpool.HTTPSConnectionPool, "ConnectionCls", _connHTTPSConnection)
 
     try:
         # unpatch botocore with awsrequest

vcr/stubs/__init__.py

@@ -47,8 +47,9 @@ def parse_headers(header_list):
 
 
 def serialize_headers(response):
+    headers = response.headers if response.msg is None else response.msg
     out = {}
-    for key, values in compat.get_headers(response.msg):
+    for key, values in compat.get_headers(headers):
         out.setdefault(key, [])
         out[key].extend(values)
     return out
@@ -67,6 +68,7 @@ class VCRHTTPResponse(HTTPResponse):
         self.version = None
         self._content = BytesIO(self.recorded_response["body"]["string"])
         self._closed = False
+        self._original_response = self  # for requests.session.Session cookie extraction
 
         headers = self.recorded_response["headers"]
         # Since we are loading a response that has already been serialized, our
@@ -143,6 +145,28 @@ class VCRHTTPResponse(HTTPResponse):
     def readable(self):
         return self._content.readable()
 
+    @property
+    def length_remaining(self):
+        return self._content.getbuffer().nbytes - self._content.tell()
+
+    def get_redirect_location(self):
+        """
+        Returns (a) redirect location string if we got a redirect
+        status code and valid location, (b) None if redirect status and
+        no location, (c) False if not a redirect status code.
+        See https://urllib3.readthedocs.io/en/stable/reference/urllib3.response.html .
+        """
+        if not (300 <= self.status <= 399):
+            return False
+        return self.getheader("Location")
+
+    @property
+    def data(self):
+        return self._content.getbuffer().tobytes()
+
+    def drain_conn(self):
+        pass
+
 
 class VCRConnection:
     # A reference to the cassette that's currently being patched in
@@ -248,12 +272,13 @@ class VCRConnection:
 
             # get the response
             response = self.real_connection.getresponse()
+            response_data = response.data if hasattr(response, "data") else response.read()
 
             # put the response into the cassette
             response = {
                 "status": {"code": response.status, "message": response.reason},
                 "headers": serialize_headers(response),
-                "body": {"string": response.read()},
+                "body": {"string": response_data},
             }
             self.cassette.append(self._vcr_request, response)
         return VCRHTTPResponse(response)

vcr/stubs/requests_stubs.py

@@ -1,6 +1,6 @@
 """Stubs for requests"""
 
-from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
+from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection
 
 from ..stubs import VCRHTTPConnection, VCRHTTPSConnection
 

vcr/stubs/urllib3_stubs.py

@@ -1,6 +1,6 @@
 """Stubs for urllib3"""
 
-from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
+from urllib3.connection import HTTPConnection, VerifiedHTTPSConnection
 
 from ..stubs import VCRHTTPConnection, VCRHTTPSConnection