From 395d2be2952869c16f356a04c10987a661110587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:26:00 +0000 Subject: [PATCH 01/16] build(deps): bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pre-commit-detect-outdated.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit-detect-outdated.yml b/.github/workflows/pre-commit-detect-outdated.yml index 8b782d1..8221654 100644 --- a/.github/workflows/pre-commit-detect-outdated.yml +++ b/.github/workflows/pre-commit-detect-outdated.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a7bb78b..d328ddf 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.12 - uses: pre-commit/action@v3.0.0 From 0782382982aaa1ce43ff39be76e92f368a7ac1ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:09:48 +0000 Subject: [PATCH 02/16] build(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d328ddf..ab82f5b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -13,7 +13,7 @@ jobs: name: Run pre-commit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: 3.12 From a3a255d60694d2da623cfe8fc0d7a344d8de2e06 Mon Sep 17 00:00:00 2001 From: pre-commit Date: Fri, 22 Dec 2023 16:04:39 +0000 Subject: [PATCH 03/16] pre-commit: Autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 471ec22..bf91ba6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.8 + rev: v0.1.9 hooks: - id: ruff args: ["--show-source"] From f1e0241673ba1a03fceb7de7ab1cf7dc1040c250 Mon Sep 17 00:00:00 2001 From: pre-commit Date: Fri, 5 Jan 2024 16:04:45 +0000 Subject: [PATCH 04/16] pre-commit: Autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf91ba6..a08e18d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.9 + rev: v0.1.11 hooks: - id: ruff args: ["--show-source"] From 5fa7010712fcd8d61a3145545ec2757b1d2d4b25 Mon Sep 17 00:00:00 2001 From: Allan Crooks Date: Sat, 20 Jan 2024 16:42:05 +0000 Subject: [PATCH 05/16] Allow HTTPX stub to read cassettes generated by other stubs. This was due to a custom format being defined in the HTTPX stub. --- .../cassettes/gzip_httpx_old_format.yaml | 41 ++++++++++++++++++ .../integration/cassettes/gzip_requests.yaml | 42 +++++++++++++++++++ tests/integration/test_httpx.py | 34 +++++++++++++-- vcr/stubs/httpx_stubs.py | 29 +++++++++---- 4 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 tests/integration/cassettes/gzip_httpx_old_format.yaml create mode 100644 tests/integration/cassettes/gzip_requests.yaml diff --git a/tests/integration/cassettes/gzip_httpx_old_format.yaml b/tests/integration/cassettes/gzip_httpx_old_format.yaml new file mode 100644 index 0000000..76f238e --- /dev/null +++ b/tests/integration/cassettes/gzip_httpx_old_format.yaml @@ -0,0 +1,41 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate, br + connection: + - keep-alive + host: + - httpbin.org + user-agent: + - python-httpx/0.23.0 + method: GET + uri: https://httpbin.org/gzip + response: + content: "{\n \"gzipped\": true, \n \"headers\": {\n \"Accept\": \"*/*\", + \n \"Accept-Encoding\": \"gzip, deflate, br\", \n \"Host\": \"httpbin.org\", + \n \"User-Agent\": \"python-httpx/0.23.0\", \n \"X-Amzn-Trace-Id\": \"Root=1-62a62a8d-5f39b5c50c744da821d6ea99\"\n + \ }, \n \"method\": \"GET\", \n \"origin\": \"146.200.25.115\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Origin: + - '*' + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Length: + - '230' + Content-Type: + - application/json + Date: + - Sun, 12 Jun 2022 18:03:57 GMT + Server: + - gunicorn/19.9.0 + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/cassettes/gzip_requests.yaml b/tests/integration/cassettes/gzip_requests.yaml new file mode 100644 index 0000000..7b9b79e --- /dev/null +++ b/tests/integration/cassettes/gzip_requests.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, br + Connection: + - keep-alive + User-Agent: + - python-requests/2.28.0 + method: GET + uri: https://httpbin.org/gzip + response: + body: + string: !!binary | + H4sIAKwrpmIA/z2OSwrCMBCG956izLIkfQSxkl2RogfQA9R2bIM1iUkqaOndnYDIrGa+/zELDB9l + LfYgg5uRwYhtj86DXKDuOrQBJKR5Cuy38kZ3pld6oHu0sqTH29QGZMnVkepgtMYuKKNJcEe0vJ3U + C4mcjI9hpaiygqaUW7ETFYGLR8frAXXE9h1Go7nD54w++FxkYp8VsDJ4IBH6E47NmVzGqUHFkn8g + rJsvp2omYs8AAAA= + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Origin: + - '*' + Connection: + - Close + Content-Encoding: + - gzip + Content-Length: + - '182' + Content-Type: + - application/json + Date: + - Sun, 12 Jun 2022 18:08:44 GMT + Server: + - Pytest-HTTPBIN/0.1.0 + status: + code: 200 + message: great +version: 1 diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 723daed..5a03aa2 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -1,5 +1,5 @@ import pytest - +import os import vcr asyncio = pytest.importorskip("asyncio") @@ -218,8 +218,8 @@ def test_work_with_gzipped_data(httpbin, do_request, yml): with vcr.use_cassette(yml) as cassette: cassette_response = do_request(headers=headers)("GET", url) - assert cassette_response.headers["content-encoding"] == "gzip" - assert cassette_response.read() + # Show we can read the content of the cassette. + assert cassette_response.json()['gzipped'] == True assert cassette.play_count == 1 @@ -287,6 +287,34 @@ def test_stream(tmpdir, httpbin, do_request): assert cassette.play_count == 1 +# Regular cassette formats support the status reason, +# but the old HTTPX cassette format does not. +@pytest.mark.parametrize( + "cassette_name,reason", + [ + ("requests", "great"), + ("httpx_old_format", "OK"), + ], +) +def test_load_gzipped(do_request, cassette_name, reason): + mydir = os.path.dirname(os.path.realpath(__file__)) + yml = f"{mydir}/cassettes/gzip_{cassette_name}.yaml" + url = "https://httpbin.org/gzip" + + with vcr.use_cassette(yml) as cassette: + cassette_response = do_request()("GET", url) + assert str(cassette_response.request.url) == url + assert cassette.play_count == 1 + + # Should be able to load up the JSON inside, + # regardless whether the content is the gzipped + # in the cassette or not. + json = cassette_response.json() + assert json["method"] == "GET", json + assert cassette_response.status_code == 200 + assert cassette_response.reason_phrase == reason + + @pytest.mark.online def test_text_content_type(tmpdir, httpbin, do_request): url = httpbin.url + "/json" diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 4d4b96a..34855c5 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -7,6 +7,8 @@ import httpx from vcr.errors import CannotOverwriteExistingCassetteException from vcr.request import Request as VcrRequest +from vcr.filters import decode_response +from vcr.serializers.compat import convert_body_to_bytes _httpx_signature = inspect.signature(httpx.Client.request) @@ -62,17 +64,30 @@ def _from_serialized_headers(headers): @patch("httpx.Response.close", MagicMock()) @patch("httpx.Response.read", MagicMock()) def _from_serialized_response(request, serialized_response, history=None): - content = serialized_response.get("content") - if isinstance(content, str): - content = content.encode("utf-8") + + # HTTPX cassette format. + if "status_code" in serialized_response: + serialized_response = decode_response(convert_body_to_bytes({ + 'headers': serialized_response['headers'], + 'body': {'string': serialized_response['content']}, + 'status': {'code': serialized_response['status_code']}, + })) + # We don't store the reason phrase in this format. + extensions = None + + # Cassette format that all other stubs use. + else: + extensions = {"reason_phrase": serialized_response["status"]["message"].encode()} + response = httpx.Response( - status_code=serialized_response.get("status_code"), + status_code=serialized_response["status"]["code"], request=request, - headers=_from_serialized_headers(serialized_response.get("headers")), - content=content, + headers=_from_serialized_headers(serialized_response["headers"]), + content=serialized_response["body"]["string"], history=history or [], + extensions=extensions, ) - response._content = content + return response From 5cf23298ac6be250c21e37d5bdb018959a2a9cd2 Mon Sep 17 00:00:00 2001 From: Allan Crooks Date: Sat, 20 Jan 2024 17:46:52 +0000 Subject: [PATCH 06/16] HTTPX stub now generates cassettes in the same format as other stubs. As part of this, I've removed the tests which inspect the data type of the response content in the cassette. That behaviour should be controlled via the inbuilt serializers. --- tests/integration/test_httpx.py | 27 ---------------------- vcr/stubs/httpx_stubs.py | 40 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 5a03aa2..d3883e2 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -314,30 +314,3 @@ def test_load_gzipped(do_request, cassette_name, reason): assert cassette_response.status_code == 200 assert cassette_response.reason_phrase == reason - -@pytest.mark.online -def test_text_content_type(tmpdir, httpbin, do_request): - url = httpbin.url + "/json" - - with vcr.use_cassette(str(tmpdir.join("json_type.yaml"))): - response = do_request()("GET", url) - - with vcr.use_cassette(str(tmpdir.join("json_type.yaml"))) as cassette: - cassette_response = do_request()("GET", url) - assert cassette_response.content == response.content - assert cassette.play_count == 1 - assert isinstance(cassette.responses[0]["content"], str) - - -@pytest.mark.online -def test_binary_content_type(tmpdir, httpbin, do_request): - url = httpbin.url + "/bytes/1024" - - with vcr.use_cassette(str(tmpdir.join("json_type.yaml"))): - response = do_request()("GET", url) - - with vcr.use_cassette(str(tmpdir.join("json_type.yaml"))) as cassette: - cassette_response = do_request()("GET", url) - assert cassette_response.content == response.content - assert cassette.play_count == 1 - assert isinstance(cassette.responses[0]["content"], bytes) diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 34855c5..477cf95 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -1,3 +1,4 @@ +import asyncio import functools import inspect import logging @@ -35,17 +36,17 @@ def _transform_headers(httpx_response): return out -def _to_serialized_response(httpx_response): - try: - content = httpx_response.content.decode("utf-8") - except UnicodeDecodeError: - content = httpx_response.content +async def _to_serialized_response(resp, aread): + + if aread: + await resp.aread() + else: + resp.read() return { - "status_code": httpx_response.status_code, - "http_version": httpx_response.http_version, - "headers": _transform_headers(httpx_response), - "content": content, + "status": dict(code=resp.status_code, message=resp.reason_phrase), + "headers": _transform_headers(resp), + "body": {"string": resp.content}, } @@ -65,17 +66,16 @@ def _from_serialized_headers(headers): @patch("httpx.Response.read", MagicMock()) def _from_serialized_response(request, serialized_response, history=None): - # HTTPX cassette format. + # Cassette format generated for HTTPX requests by older versions of + # vcrpy. We restructure the content to resemble what a regular + # cassette looks like. if "status_code" in serialized_response: serialized_response = decode_response(convert_body_to_bytes({ 'headers': serialized_response['headers'], 'body': {'string': serialized_response['content']}, 'status': {'code': serialized_response['status_code']}, })) - # We don't store the reason phrase in this format. extensions = None - - # Cassette format that all other stubs use. else: extensions = {"reason_phrase": serialized_response["status"]["message"].encode()} @@ -113,17 +113,17 @@ def _shared_vcr_send(cassette, real_send, *args, **kwargs): return vcr_request, None -def _record_responses(cassette, vcr_request, real_response): +async def _record_responses(cassette, vcr_request, real_response, aread): for past_real_response in real_response.history: past_vcr_request = _make_vcr_request(past_real_response.request) - cassette.append(past_vcr_request, _to_serialized_response(past_real_response)) + cassette.append(past_vcr_request, await _to_serialized_response(past_real_response, aread)) if real_response.history: # If there was a redirection keep we want the request which will hold the # final redirect value vcr_request = _make_vcr_request(real_response.request) - cassette.append(vcr_request, _to_serialized_response(real_response)) + cassette.append(vcr_request, await _to_serialized_response(real_response, aread)) return real_response @@ -141,8 +141,8 @@ async def _async_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = await real_send(*args, **kwargs) - await real_response.aread() - return _record_responses(cassette, vcr_request, real_response) + await _record_responses(cassette, vcr_request, real_response, aread=True) + return real_response def async_vcr_send(cassette, real_send): @@ -161,8 +161,8 @@ def _sync_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = real_send(*args, **kwargs) - real_response.read() - return _record_responses(cassette, vcr_request, real_response) + asyncio.run(_record_responses(cassette, vcr_request, real_response, aread=False)) + return real_response def sync_vcr_send(cassette, real_send): From c5487384ee14d5a508cb5aab41aa9e88cb6780e2 Mon Sep 17 00:00:00 2001 From: Allan Crooks Date: Sun, 21 Jan 2024 15:16:07 +0000 Subject: [PATCH 07/16] Fix handling of encoded content in HTTPX stub. Also copied over and adjusted some of the tests from test_requests.py relating to gzipped handling to show that the HTTPX stub is behaving in a consistent way to how the requests stub is. --- tests/integration/test_httpx.py | 64 ++++++++++++++++++++++++--------- vcr/stubs/httpx_stubs.py | 20 ++++++++--- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index d3883e2..e508f6d 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -5,6 +5,7 @@ import vcr asyncio = pytest.importorskip("asyncio") httpx = pytest.importorskip("httpx") +from ..assertions import assert_is_json_bytes @pytest.fixture(params=["https", "http"]) def scheme(request): @@ -207,22 +208,6 @@ def test_redirect(httpbin, yml, do_request): assert cassette_response.request.headers.items() == response.request.headers.items() -@pytest.mark.online -def test_work_with_gzipped_data(httpbin, do_request, yml): - url = httpbin.url + "/gzip?foo=bar" - headers = {"accept-encoding": "deflate, gzip"} - - with vcr.use_cassette(yml): - do_request(headers=headers)("GET", url) - - with vcr.use_cassette(yml) as cassette: - cassette_response = do_request(headers=headers)("GET", url) - - # Show we can read the content of the cassette. - assert cassette_response.json()['gzipped'] == True - assert cassette.play_count == 1 - - @pytest.mark.online @pytest.mark.parametrize("url", ["https://github.com/kevin1024/vcrpy/issues/" + str(i) for i in range(3, 6)]) def test_simple_fetching(do_request, yml, url): @@ -296,7 +281,7 @@ def test_stream(tmpdir, httpbin, do_request): ("httpx_old_format", "OK"), ], ) -def test_load_gzipped(do_request, cassette_name, reason): +def test_load_cassette_format(do_request, cassette_name, reason): mydir = os.path.dirname(os.path.realpath(__file__)) yml = f"{mydir}/cassettes/gzip_{cassette_name}.yaml" url = "https://httpbin.org/gzip" @@ -314,3 +299,48 @@ def test_load_gzipped(do_request, cassette_name, reason): assert cassette_response.status_code == 200 assert cassette_response.reason_phrase == reason + +def test_gzip__decode_compressed_response_false(tmpdir, httpbin, do_request): + """ + Ensure that httpx is able to automatically decompress the response body. + """ + for _ in range(2): # one for recording, one for re-playing + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))) as cassette: + response = do_request()("GET", httpbin + "/gzip") + assert response.headers["content-encoding"] == "gzip" # i.e. not removed + # The content stored in the cassette should be gzipped. + assert cassette.responses[0]["body"]["string"][:2] == b"\x1f\x8b" + assert_is_json_bytes(response.content) # i.e. uncompressed bytes + + +def test_gzip__decode_compressed_response_true(do_request, tmpdir, httpbin): + url = httpbin + "/gzip" + + expected_response = do_request()("GET", url) + expected_content = expected_response.content + assert expected_response.headers["content-encoding"] == "gzip" # self-test + + with vcr.use_cassette( + str(tmpdir.join("decode_compressed.yaml")), + decode_compressed_response=True, + ) as cassette: + r = do_request()("GET", url) + assert r.headers["content-encoding"] == "gzip" # i.e. not removed + content_length = r.headers["content-length"] + assert r.content == expected_content + + # Has the cassette body been decompressed? + cassette_response_body = cassette.responses[0]["body"]["string"] + assert isinstance(cassette_response_body, str) + + # Content should be JSON. + assert cassette_response_body[0:1] == "{" + + with vcr.use_cassette(str(tmpdir.join("decode_compressed.yaml")), decode_compressed_response=True): + r = httpx.get(url) + assert "content-encoding" not in r.headers # i.e. removed + assert r.content == expected_content + + # As the content is uncompressed, it should have a bigger + # length than the compressed version. + assert r.headers["content-length"] > content_length diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 477cf95..8fc63e7 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -38,17 +38,27 @@ def _transform_headers(httpx_response): async def _to_serialized_response(resp, aread): - if aread: - await resp.aread() - else: - resp.read() + # The content shouldn't already have been read in by HTTPX. + assert not hasattr(resp, "_decoder") - return { + # Retrieve the content, but without decoding it. + with patch.dict(resp.headers, {"Content-Encoding": ""}): + if aread: + await resp.aread() + else: + resp.read() + + result = { "status": dict(code=resp.status_code, message=resp.reason_phrase), "headers": _transform_headers(resp), "body": {"string": resp.content}, } + # As the content wasn't decoded, we restore the response to a state which + # will be capable of decoding the content for the consumer. + del resp._decoder + resp._content = resp._get_content_decoder().decode(resp.content) + return result def _from_serialized_headers(headers): """ From 54bc6467eba4c2c9f59a74310b3dbafa17658531 Mon Sep 17 00:00:00 2001 From: Allan Crooks Date: Sun, 21 Jan 2024 15:50:46 +0000 Subject: [PATCH 08/16] Run linters. --- tests/integration/test_httpx.py | 7 +++++-- vcr/stubs/httpx_stubs.py | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index e508f6d..d261927 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -1,11 +1,14 @@ -import pytest import os + +import pytest + import vcr +from ..assertions import assert_is_json_bytes + asyncio = pytest.importorskip("asyncio") httpx = pytest.importorskip("httpx") -from ..assertions import assert_is_json_bytes @pytest.fixture(params=["https", "http"]) def scheme(request): diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 8fc63e7..759cb72 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -7,8 +7,8 @@ from unittest.mock import MagicMock, patch import httpx from vcr.errors import CannotOverwriteExistingCassetteException -from vcr.request import Request as VcrRequest from vcr.filters import decode_response +from vcr.request import Request as VcrRequest from vcr.serializers.compat import convert_body_to_bytes _httpx_signature = inspect.signature(httpx.Client.request) @@ -37,7 +37,6 @@ def _transform_headers(httpx_response): async def _to_serialized_response(resp, aread): - # The content shouldn't already have been read in by HTTPX. assert not hasattr(resp, "_decoder") @@ -49,7 +48,7 @@ async def _to_serialized_response(resp, aread): resp.read() result = { - "status": dict(code=resp.status_code, message=resp.reason_phrase), + "status": {"code": resp.status_code, "message": resp.reason_phrase}, "headers": _transform_headers(resp), "body": {"string": resp.content}, } @@ -60,6 +59,7 @@ async def _to_serialized_response(resp, aread): resp._content = resp._get_content_decoder().decode(resp.content) return result + def _from_serialized_headers(headers): """ httpx accepts headers as list of tuples of header key and value. @@ -75,16 +75,19 @@ def _from_serialized_headers(headers): @patch("httpx.Response.close", MagicMock()) @patch("httpx.Response.read", MagicMock()) def _from_serialized_response(request, serialized_response, history=None): - # Cassette format generated for HTTPX requests by older versions of # vcrpy. We restructure the content to resemble what a regular # cassette looks like. if "status_code" in serialized_response: - serialized_response = decode_response(convert_body_to_bytes({ - 'headers': serialized_response['headers'], - 'body': {'string': serialized_response['content']}, - 'status': {'code': serialized_response['status_code']}, - })) + serialized_response = decode_response( + convert_body_to_bytes( + { + "headers": serialized_response["headers"], + "body": {"string": serialized_response["content"]}, + "status": {"code": serialized_response["status_code"]}, + }, + ), + ) extensions = None else: extensions = {"reason_phrase": serialized_response["status"]["message"].encode()} From 1677154f0419bd1192c722f36475ca087d1dd06a Mon Sep 17 00:00:00 2001 From: pre-commit Date: Fri, 19 Jan 2024 16:04:51 +0000 Subject: [PATCH 09/16] pre-commit: Autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a08e18d..688c4d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.11 + rev: v0.1.13 hooks: - id: ruff args: ["--show-source"] From 53f686aa5b2272d7a507eb975ba9a96807a2d921 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:28:16 -0300 Subject: [PATCH 10/16] Refactor test to not use tox.ini file --- tests/integration/test_requests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index 48590fc..10cbd21 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -265,7 +265,7 @@ def test_nested_cassettes_with_session_created_before_nesting(httpbin_both, tmpd def test_post_file(tmpdir, httpbin_both): """Ensure that we handle posting a file.""" url = httpbin_both + "/post" - with vcr.use_cassette(str(tmpdir.join("post_file.yaml"))) as cass, open("tox.ini", "rb") as f: + with vcr.use_cassette(str(tmpdir.join("post_file.yaml"))) as cass, open(".editorconfig", "rb") as f: original_response = requests.post(url, f).content # This also tests that we do the right thing with matching the body when they are files. @@ -273,10 +273,10 @@ def test_post_file(tmpdir, httpbin_both): str(tmpdir.join("post_file.yaml")), match_on=("method", "scheme", "host", "port", "path", "query", "body"), ) as cass: - with open("tox.ini", "rb") as f: - tox_content = f.read() - assert cass.requests[0].body.read() == tox_content - with open("tox.ini", "rb") as f: + with open(".editorconfig", "rb") as f: + editorconfig = f.read() + assert cass.requests[0].body.read() == editorconfig + with open(".editorconfig", "rb") as f: new_response = requests.post(url, f).content assert original_response == new_response From 0594de9b3ed98b3860d78f6f50bd787d82cd6781 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:28:42 -0300 Subject: [PATCH 11/16] Remove tox.ini from MANIFEST.in file --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9fc7449..2606d12 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ include README.rst include LICENSE.txt -include tox.ini recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] From abbb50135f07e709f1864f190178176cfaf9bae5 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:29:27 -0300 Subject: [PATCH 12/16] Organize dependencies for tests and extras requires --- setup.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 71ba6a2..7a49061 100644 --- a/setup.py +++ b/setup.py @@ -57,24 +57,29 @@ install_requires = [ "urllib3 <2; platform_python_implementation =='PyPy'", ] -tests_require = [ - "aiohttp", - "boto3", - "httplib2", - "httpx", - "pytest", - "pytest-aiohttp", - "pytest-httpbin", - "requests>=2.16.2", - "tornado", - # Needed to un-break httpbin 0.7.0. For httpbin >=0.7.1 and after, - # this pin and the dependency itself can be removed, provided - # that the related bug in httpbin has been fixed: - # https://github.com/kevin1024/vcrpy/issues/645#issuecomment-1562489489 - # https://github.com/postmanlabs/httpbin/issues/673 - # https://github.com/postmanlabs/httpbin/pull/674 - "Werkzeug==2.0.3", -] +extras_require = { + "tests": [ + "aiohttp", + "boto3", + "httplib2", + "httpx", + "pytest-aiohttp", + "pytest-asyncio", + "pytest-cov", + "pytest-httpbin", + "pytest", + "requests>=2.22.0", + "tornado", + "urllib3", + # Needed to un-break httpbin 0.7.0. For httpbin >=0.7.1 and after, + # this pin and the dependency itself can be removed, provided + # that the related bug in httpbin has been fixed: + # https://github.com/kevin1024/vcrpy/issues/645#issuecomment-1562489489 + # https://github.com/postmanlabs/httpbin/issues/673 + # https://github.com/postmanlabs/httpbin/pull/674 + "Werkzeug==2.0.3", + ], +} setup( name="vcrpy", @@ -89,7 +94,8 @@ setup( python_requires=">=3.8", install_requires=install_requires, license="MIT", - tests_require=tests_require, + extras_require=extras_require, + tests_require=extras_require["tests"], classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", From e37fc9ab6ea6b4ae27f4ac3ff6c8e2a727662212 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:30:29 -0300 Subject: [PATCH 13/16] Refactor ci main workflow to use matrix to run tests without tox --- .github/workflows/main.yml | 41 ++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53f1488..a9d8202 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,30 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8", "pypy-3.9", "pypy-3.10"] + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "pypy-3.8" + - "pypy-3.9" + - "pypy-3.10" + urllib3-requirement: + - "urllib3>=2" + - "urllib3<2" + + exclude: + - python-version: "3.8" + urllib3-requirement: "urllib3>=2" + - python-version: "pypy-3.8" + urllib3-requirement: "urllib3>=2" + - python-version: "3.9" + urllib3-requirement: "urllib3>=2" + - python-version: "pypy-3.9" + urllib3-requirement: "urllib3>=2" + - python-version: "pypy-3.10" + urllib3-requirement: "urllib3>=2" steps: - uses: actions/checkout@v4 @@ -22,22 +45,24 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip - name: Install project dependencies run: | - pip3 install --upgrade pip - pip3 install codecov tox tox-gh-actions + pip install --upgrade pip + pip install codecov '.[tests]' '${{ matrix.urllib3-requirement }}' + pip check - - name: Run online tests with tox - run: tox -- -m online + - name: Run online tests + run: ./runtests.sh --cov=./vcr --cov-branch --cov-report=xml --cov-append -m online - - name: Run offline tests with tox with no access to the Internet + - name: Run offline tests with no access to the Internet run: | # We're using unshare to take Internet access - # away from tox so that we'll notice whenever some new test + # away so that we'll notice whenever some new test # is missing @pytest.mark.online decoration in the future unshare --map-root-user --net -- \ - sh -c 'ip link set lo up; tox -- -m "not online"' + sh -c 'ip link set lo up; ./runtests.sh --cov=./vcr --cov-branch --cov-report=xml --cov-append -m "not online"' - name: Run coverage run: codecov From cb77cb8f691c2458ebdef9040fd46b3f136b4d2a Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:30:49 -0300 Subject: [PATCH 14/16] Remove tox.ini file --- tox.ini | 74 --------------------------------------------------------- 1 file changed, 74 deletions(-) delete mode 100644 tox.ini diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 2992315..0000000 --- a/tox.ini +++ /dev/null @@ -1,74 +0,0 @@ -[tox] -skip_missing_interpreters=true -envlist = - cov-clean, - {py38,py39,py310,py311,py312}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3,aiohttp,httpx}, - {py310,py311,py312}-{requests-urllib3-2,urllib3-2}, - {pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3}, - #{py310}-httpx019, - cov-report - - -[gh-actions] -python = - 3.8: py38 - 3.9: py39 - 3.10: py310 - 3.11: py311 - 3.12: py312 - pypy-3: pypy3 - -# Coverage environment tasks: cov-clean and cov-report -# https://pytest-cov.readthedocs.io/en/latest/tox.html -[testenv:cov-clean] -deps = coverage -skip_install=true -commands = coverage erase - -[testenv:cov-report] -deps = coverage -skip_install=true -commands = - coverage html - coverage report --fail-under=90 - -[testenv] -# Need to use develop install so that paths -# for aggregate code coverage combine -usedevelop=true -commands = - ./runtests.sh --cov=./vcr --cov-branch --cov-report=xml --cov-append {posargs} -allowlist_externals = - ./runtests.sh -deps = - Werkzeug==2.0.3 - pytest - pytest-httpbin>=1.0.1 - pytest-cov - PyYAML - ipaddress - requests: requests>=2.22.0 - httplib2: httplib2 - urllib3-1: urllib3<2 - urllib3-2: urllib3<3 - boto3: boto3 - aiohttp: aiohttp - aiohttp: pytest-asyncio - aiohttp: pytest-aiohttp - httpx: httpx - {py38,py39,py310}-{httpx}: httpx - {py38,py39,py310}-{httpx}: pytest-asyncio - httpx: httpx>0.19 - httpx019: httpx==0.19 - {py38,py39,py310}-{httpx}: pytest-asyncio -depends = - {py38,py39,py310,py311,py312,pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3},{py310,py311,py312}-{requests-urllib3-2,urllib3-2},{py38,py39,py310,py311,py312}-{aiohttp},{py38,py39,py310,py311,py312}-{httpx}: cov-clean - cov-report: {py38,py39,py310,py311,py312,pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3},{py310,py311,py312}-{requests-urllib3-2,urllib3-2},{py38,py39,py310,py311,py312}-{aiohttp} -passenv = - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION - AWS_SECRET_ACCESS_KEY -setenv = - # workaround for broken C extension in aiohttp - # see: https://github.com/aio-libs/aiohttp/issues/7229 - py312: AIOHTTP_NO_EXTENSIONS=1 From f9b69d8da7657740625f3493dd71ffc88cdd5ad3 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:40:49 -0300 Subject: [PATCH 15/16] Remove tox reference from runtests.sh file --- runtests.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/runtests.sh b/runtests.sh index 9e40a2d..5e00b8b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,7 +1,5 @@ #!/bin/bash -# https://blog.ionelmc.ro/2015/04/14/tox-tricks-and-patterns/#when-it-inevitably-leads-to-shell-scripts -# If you are getting an INVOCATION ERROR for this script then there is -# a good chance you are running on Windows. -# You can and should use WSL for running tox on Windows when it calls bash scripts. +# If you are getting an INVOCATION ERROR for this script then there is a good chance you are running on Windows. +# You can and should use WSL for running tests on Windows when it calls bash scripts. REQUESTS_CA_BUNDLE=`python3 -m pytest_httpbin.certs` exec pytest "$@" From 62fe272a8e0553ee08dd9cf1753ecf99218a20c5 Mon Sep 17 00:00:00 2001 From: Jair Henrique Date: Mon, 11 Dec 2023 17:41:17 -0300 Subject: [PATCH 16/16] Remove tox reference from contributing docs --- docs/contributing.rst | 43 ++++++++++--------------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 6cbc41d..5716238 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -83,39 +83,21 @@ The PR reviewer is a second set of eyes to see if: Running VCR's test suite ------------------------ -The tests are all run automatically on `Travis -CI `__, but you can also run them -yourself using `pytest `__ and -`Tox `__. +The tests are all run automatically on `Github Actions CI `__, +but you can also run them yourself using `pytest `__. -Tox will automatically run them in all environments VCR.py supports if they are available on your `PATH`. Alternatively you can use `tox-pyenv `_ with -`pyenv `_. -We recommend you read the documentation for each and see the section further below. - -The test suite is pretty big and slow, but you can tell tox to only run specific tests like this:: - - tox -e {pyNN}-{HTTP_LIBRARY} -- - - tox -e py38-requests -- -v -k "'test_status_code or test_gzip'" - tox -e py38-requests -- -v --last-failed - -This will run only tests that look like ``test_status_code`` or -``test_gzip`` in the test suite, and only in the python 3.8 environment -that has ``requests`` installed. - -Also, in order for the boto3 tests to run, you will need an AWS key. +In order for the boto3 tests to run, you will need an AWS key. Refer to the `boto3 documentation `__ for how to set this up. I have marked the boto3 tests as optional in Travis so you don't have to worry about them failing if you submit a pull request. -Using PyEnv with VCR's test suite +Using Pyenv with VCR's test suite --------------------------------- -PyEnv is a tool for managing multiple installation of python on your system. +Pyenv is a tool for managing multiple installation of python on your system. See the full documentation at their `github `_ -but we are also going to use `tox-pyenv `_ in this example:: git clone https://github.com/pyenv/pyenv ~/.pyenv @@ -126,26 +108,21 @@ in this example:: # Setup shim paths eval "$(pyenv init -)" - # Setup your local system tox tooling - pip3 install tox tox-pyenv - # Install supported versions (at time of writing), this does not activate them - pyenv install 3.8.0 pypy3.8 + pyenv install 3.12.0 pypy3.10 # This activates them - pyenv local 3.8.0 pypy3.8 + pyenv local 3.12.0 pypy3.10 # Run the whole test suite - tox - - # Run the whole test suite or just part of it - tox -e py38-requests + pip install .[test] + ./run_tests.sh Troubleshooting on MacOSX ------------------------- -If you have this kind of error when running tox : +If you have this kind of error when running tests : .. code:: python