From 9daf301deb2152eca8b9fb354b0dbf0bade613b3 Mon Sep 17 00:00:00 2001 From: Aliaksandr Buhayeu Date: Wed, 2 Dec 2015 16:33:58 +0300 Subject: [PATCH] Fix for Serialization errors with JSON adapter This patch aims to fix the issue#222, where json data in request can not be serialized because of TypeError in py3 --- tests/unit/test_serialize.py | 50 ++++++++++++++++++++++++++++++- vcr/serialize.py | 2 +- vcr/serializers/compat.py | 48 +++++++++++++++++++++-------- vcr/serializers/jsonserializer.py | 15 ++++++---- 4 files changed, 96 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_serialize.py b/tests/unit/test_serialize.py index 3e03adb..cf3f0a8 100644 --- a/tests/unit/test_serialize.py +++ b/tests/unit/test_serialize.py @@ -2,7 +2,8 @@ import pytest from vcr.compat import mock -from vcr.serialize import deserialize +from vcr.request import Request +from vcr.serialize import deserialize, serialize from vcr.serializers import yamlserializer, jsonserializer @@ -83,3 +84,50 @@ def test_deserialize_py2py3_yaml_cassette(tmpdir, req_body, expect): def test_serialize_constructs_UnicodeDecodeError(mock_dumps): with pytest.raises(UnicodeDecodeError): jsonserializer.serialize({}) + + +def test_serialize_empty_request(): + request = Request( + method='POST', + uri='http://localhost/', + body='', + headers={}, + ) + + serialize( + {'requests': [request], 'responses': [{}]}, + jsonserializer + ) + + +def test_serialize_json_request(): + request = Request( + method='POST', + uri='http://localhost/', + body="{'hello': 'world'}", + headers={}, + ) + + serialize( + {'requests': [request], 'responses': [{}]}, + jsonserializer + ) + + +def test_serialize_binary_request(): + msg = "Does this HTTP interaction contain binary data?" + + request = Request( + method='POST', + uri='http://localhost/', + body=b'\x8c', + headers={}, + ) + + try: + serialize( + {'requests': [request], 'responses': [{}]}, + jsonserializer + ) + except (UnicodeDecodeError, TypeError) as exc: + assert msg in str(exc) diff --git a/vcr/serialize.py b/vcr/serialize.py index bf66911..a9c04d6 100644 --- a/vcr/serialize.py +++ b/vcr/serialize.py @@ -50,7 +50,7 @@ def deserialize(cassette_string, serializer): def serialize(cassette_dict, serializer): interactions = ([{ - 'request': request._to_dict(), + 'request': compat.convert_to_unicode(request._to_dict()), 'response': compat.convert_to_unicode(response), } for request, response in zip( cassette_dict['requests'], diff --git a/vcr/serializers/compat.py b/vcr/serializers/compat.py index 2b9e46b..0fcc583 100644 --- a/vcr/serializers/compat.py +++ b/vcr/serializers/compat.py @@ -17,7 +17,7 @@ def convert_body_to_bytes(resp): By default yaml serializes to utf-8 encoded bytestrings. When this cassette is loaded by python3, it's automatically decoded - into unicode strings. This makes sure that it stays a bytestring, since + into unicode strings. This makes sure that it stays a bytestring, since that's what all the internal httplib machinery is expecting. For more info on py3 yaml: @@ -37,19 +37,43 @@ def convert_body_to_bytes(resp): return resp +def _convert_string_to_unicode(string): + """ + If the string is bytes, decode it to a string (for python3 support) + """ + result = string + + try: + if string is not None and not isinstance(string, six.text_type): + result = string.decode('utf-8') + except (TypeError, UnicodeDecodeError, AttributeError): + # Sometimes the string actually is binary or StringIO object, + # so if you can't decode it, just give up. + pass + + return result + + def convert_body_to_unicode(resp): """ - If the request body is bytes, decode it to a string (for python3 support) + If the request or responses body is bytes, decode it to a string + (for python3 support) """ - try: - if not isinstance(resp['body']['string'], six.text_type): - resp['body']['string'] = resp['body']['string'].decode('utf-8') - except (KeyError, TypeError, UnicodeDecodeError): - # The thing we were converting either wasn't a dictionary or didn't - # have the keys we were expecting. Some of the tests just serialize - # and deserialize a string. + if type(resp) is not dict: + # Some of the tests just serialize and deserialize a string. + return _convert_string_to_unicode(resp) + else: + body = resp.get('body') + + if body is not None: + try: + body['string'] = _convert_string_to_unicode( + body['string'] + ) + except (KeyError, TypeError, AttributeError): + # The thing we were converting either wasn't a dictionary or + # didn't have the keys we were expecting. + # For example request object has no 'string' key. + resp['body'] = _convert_string_to_unicode(body) - # Also, sometimes the thing actually is binary, so if you can't decode - # it, just give up. - pass return resp diff --git a/vcr/serializers/jsonserializer.py b/vcr/serializers/jsonserializer.py index b701040..afac943 100644 --- a/vcr/serializers/jsonserializer.py +++ b/vcr/serializers/jsonserializer.py @@ -9,16 +9,21 @@ def deserialize(cassette_string): def serialize(cassette_dict): + error_message = ( + "Does this HTTP interaction contain binary data? " + "If so, use a different serializer (like the yaml serializer) " + "for this request?" + ) + try: return json.dumps(cassette_dict, indent=4) - except UnicodeDecodeError as original: + except UnicodeDecodeError as original: # py2 raise UnicodeDecodeError( original.encoding, b"Error serializing cassette to JSON", original.start, original.end, - original.args[-1] + - ("Does this HTTP interaction contain binary data? " - "If so, use a different serializer (like the yaml serializer) " - "for this request?") + original.args[-1] + error_message ) + except TypeError as original: # py3 + raise TypeError(error_message)