diff --git a/docs/advanced.rst b/docs/advanced.rst index 198a3d2..ed5aa08 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -97,8 +97,12 @@ Create your own method with the following signature def my_matcher(r1, r2): -Your method receives the two requests and must return ``True`` if they -match, ``False`` if they don't. +Your method receives the two requests and can return : + +- Use an ``assert`` statement in the matcher, then we have ``None`` if they match, raise an `AssertionError`` if they don't. +- A boolean, ``True`` if they match, ``False`` if they don't. + +Note : You should use an ``assert`` statement in order to have feedback when a matcher is failing. Finally, register your method with VCR to use your new request matcher. @@ -107,7 +111,7 @@ Finally, register your method with VCR to use your new request matcher. import vcr def jurassic_matcher(r1, r2): - return r1.uri == r2.uri and 'JURASSIC PARK' in r1.body + assert r1.uri == r2.uri and 'JURASSIC PARK' in r1.body my_vcr = vcr.VCR() my_vcr.register_matcher('jurassic', jurassic_matcher) diff --git a/docs/usage.rst b/docs/usage.rst index e7d6fe2..e4223d2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -95,3 +95,9 @@ Unittest Integration While it's possible to use the context manager or decorator forms with unittest, there's also a ``VCRTestCase`` provided separately by `vcrpy-unittest `__. + +Pytest Integration +------------------ + +A Pytest plugin is available here : `pytest-vcr +`__. diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 4541530..01d8c70 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -317,3 +317,51 @@ def test_use_as_decorator_on_generator(): yield 2 assert list(test_function()) == [1, 2] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_one_similar_request(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ([], [("method", "failed : method"), ("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [(2, ["method", "path"], [("query", "failed : query")])] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_no_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_many_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method", "path"], [("query", "failed : query")]), + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, 'response') + result = cassette.find_requests_with_most_matches("fake request") + assert result == [ + (1, ["method", "path"], [("query", "failed : query")]), + (3, ["method", "path"], [("query", "failed : query")]) + ] diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 9889b4f..604b650 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -1,4 +1,5 @@ import itertools +from vcr.compat import mock import pytest @@ -21,20 +22,22 @@ REQUESTS = { def assert_matcher(matcher_name): matcher = getattr(matchers, matcher_name) for k1, k2 in itertools.permutations(REQUESTS, 2): - matched = matcher(REQUESTS[k1], REQUESTS[k2]) - if matcher_name in {k1, k2}: - assert not matched + expecting_assertion_error = matcher_name in {k1, k2} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matcher(REQUESTS[k1], REQUESTS[k2]) else: - assert matched + assert matcher(REQUESTS[k1], REQUESTS[k2]) is None def test_uri_matcher(): for k1, k2 in itertools.permutations(REQUESTS, 2): - matched = matchers.uri(REQUESTS[k1], REQUESTS[k2]) - if {k1, k2} != {'base', 'method'}: - assert not matched + expecting_assertion_error = {k1, k2} != {"base", "method"} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matchers.uri(REQUESTS[k1], REQUESTS[k2]) else: - assert matched + assert matchers.uri(REQUESTS[k1], REQUESTS[k2]) is None req1_body = (b"test" @@ -107,7 +110,7 @@ req2_body = (b"test" ) ]) def test_body_matcher_does_match(r1, r2): - assert matchers.body(r1, r2) + assert matchers.body(r1, r2) is None @pytest.mark.parametrize("r1, r2", [ @@ -135,25 +138,132 @@ def test_body_matcher_does_match(r1, r2): ) ]) def test_body_match_does_not_match(r1, r2): - assert not matchers.body(r1, r2) + with pytest.raises(AssertionError): + matchers.body(r1, r2) def test_query_matcher(): - req1 = request.Request('GET', 'http://host.com/?a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {}) - assert matchers.query(req1, req2) + req1 = request.Request("GET", "http://host.com/?a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?c=d&a=b", "", {}) + assert matchers.query(req1, req2) is None - req1 = request.Request('GET', 'http://host.com/?a=b&a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com/?a=b&c=d&a=b', '', {}) - req3 = request.Request('GET', 'http://host.com/?c=d&a=b&a=b', '', {}) - assert matchers.query(req1, req2) - assert matchers.query(req1, req3) + req1 = request.Request("GET", "http://host.com/?a=b&a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?a=b&c=d&a=b", "", {}) + req3 = request.Request("GET", "http://host.com/?c=d&a=b&a=b", "", {}) + assert matchers.query(req1, req2) is None + assert matchers.query(req1, req3) is None -def test_metchers(): - assert_matcher('method') - assert_matcher('scheme') - assert_matcher('host') - assert_matcher('port') - assert_matcher('path') - assert_matcher('query') +def test_matchers(): + assert_matcher("method") + assert_matcher("scheme") + assert_matcher("host") + assert_matcher("port") + assert_matcher("path") + assert_matcher("query") + + +def test_evaluate_matcher_does_match(): + def bool_matcher(r1, r2): + return True + + def assertion_matcher(r1, r2): + assert 1 == 1 + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is True + assert assertion_msg is None + + +def test_evaluate_matcher_does_not_match(): + def bool_matcher(r1, r2): + return False + + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError() + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is False + assert not assertion_msg + + +def test_evaluate_matcher_does_not_match_with_assert_message(): + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError("Failing matcher") + + r1, r2 = None, None + match, assertion_msg = matchers._evaluate_matcher(assertion_matcher, r1, r2) + assert match is False + assert assertion_msg == "Failing matcher" + + +def test_get_assertion_message(): + assert matchers.get_assertion_message(None) == "" + assert matchers.get_assertion_message("") == "" + + +def test_get_assertion_message_with_details(): + assertion_msg = "q1=1 != q2=1" + expected = ( + "--------------- DETAILS ---------------\n" + "{}\n" + "----------------------------------------\n".format(assertion_msg) + ) + assert matchers.get_assertion_message(assertion_msg) == expected + + +@pytest.mark.parametrize( + "r1, r2, expected_successes, expected_failures", + [ + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("GET", "http://host.com/p?a=b", "", {}), + ["method", "path"], + [], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/p?a=b", "", {}), + ["path"], + ["method"], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/path?a=b", "", {}), + [], + ["method", "path"], + ), + ], +) +def test_get_matchers_results(r1, r2, expected_successes, expected_failures): + successes, failures = matchers.get_matchers_results( + r1, r2, [matchers.method, matchers.path] + ) + assert successes == expected_successes + assert len(failures) == len(expected_failures) + for i, expected_failure in enumerate(expected_failures): + assert failures[i][0] == expected_failure + assert failures[i][1] is not None + + +@mock.patch("vcr.matchers.get_matchers_results") +@pytest.mark.parametrize( + "successes, failures, expected_match", + [ + (["method", "path"], [], True), + (["method"], ["path"], False), + ([], ["method", "path"], False), + ], +) +def test_requests_match(mock_get_matchers_results, successes, failures, expected_match): + mock_get_matchers_results.return_value = (successes, failures) + r1 = request.Request("GET", "http://host.com/p?a=b", "", {}) + r2 = request.Request("GET", "http://host.com/p?a=b", "", {}) + match = matchers.requests_match(r1, r2, [matchers.method, matchers.path]) + assert match is expected_match diff --git a/vcr/cassette.py b/vcr/cassette.py index 79d8816..7ba092c 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -8,7 +8,7 @@ import wrapt from .compat import contextlib from .errors import UnhandledHTTPRequestError -from .matchers import requests_match, uri, method +from .matchers import requests_match, uri, method, get_matchers_results from .patch import CassettePatcherBuilder from .serializers import yamlserializer from .persisters.filesystem import FilesystemPersister @@ -290,6 +290,38 @@ class Cassette(object): def rewind(self): self.play_counts = collections.Counter() + def find_requests_with_most_matches(self, request): + """ + Get the most similar request(s) stored in the cassette + of a given request as a list of tuples like this: + - the request object + - the successful matchers as string + - the failed matchers and the related assertion message with the difference details as strings tuple + + This is useful when a request failed to be found, + we can get the similar request(s) in order to know what have changed in the request parts. + """ + best_matches = [] + request = self._before_record_request(request) + for index, (stored_request, response) in enumerate(self.data): + successes, fails = get_matchers_results(request, stored_request, self._match_on) + best_matches.append((len(successes), stored_request, successes, fails)) + best_matches.sort(key=lambda t: t[0], reverse=True) + # Get the first best matches (multiple if equal matches) + final_best_matches = [] + previous_nb_success = best_matches[0][0] + for best_match in best_matches: + nb_success = best_match[0] + # Do not keep matches that have 0 successes, + # it means that the request is totally different from + # the ones stored in the cassette + if nb_success < 1 or previous_nb_success != nb_success: + break + previous_nb_success = nb_success + final_best_matches.append(best_match[1:]) + + return final_best_matches + def _as_dict(self): return {"requests": self.requests, "responses": self.responses} diff --git a/vcr/errors.py b/vcr/errors.py index bdc9701..f762e8c 100644 --- a/vcr/errors.py +++ b/vcr/errors.py @@ -1,5 +1,30 @@ class CannotOverwriteExistingCassetteException(Exception): - pass + def __init__(self, *args, **kwargs): + message = self._get_message(kwargs["cassette"], kwargs["failed_request"]) + super(CannotOverwriteExistingCassetteException, self).__init__(message) + + def _get_message(self, cassette, failed_request): + """Get the final message related to the exception""" + # Get the similar requests in the cassette that + # have match the most with the request. + best_matches = cassette.find_requests_with_most_matches(failed_request) + # Build a comprehensible message to put in the exception. + best_matches_msg = "" + for best_match in best_matches: + request, _, failed_matchers_assertion_msgs = best_match + best_matches_msg += "Similar request found : (%r).\n" % request + for failed_matcher, assertion_msg in failed_matchers_assertion_msgs: + best_matches_msg += "Matcher failed : %s\n" "%s\n" % ( + failed_matcher, + assertion_msg, + ) + return ( + "Can't overwrite existing cassette (%r) in " + "your current record mode (%r).\n" + "No match for the request (%r) was found.\n" + "%s" + % (cassette._path, cassette.record_mode, failed_request, best_matches_msg) + ) class UnhandledHTTPRequestError(KeyError): diff --git a/vcr/matchers.py b/vcr/matchers.py index ad04a45..1714018 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -8,35 +8,47 @@ log = logging.getLogger(__name__) def method(r1, r2): - return r1.method == r2.method + assert r1.method == r2.method, "{} != {}".format(r1.method, r2.method) def uri(r1, r2): - return r1.uri == r2.uri + assert r1.uri == r2.uri, "{} != {}".format(r1.uri, r2.uri) def host(r1, r2): - return r1.host == r2.host + assert r1.host == r2.host, "{} != {}".format(r1.host, r2.host) def scheme(r1, r2): - return r1.scheme == r2.scheme + assert r1.scheme == r2.scheme, "{} != {}".format(r1.scheme, r2.scheme) def port(r1, r2): - return r1.port == r2.port + assert r1.port == r2.port, "{} != {}".format(r1.port, r2.port) def path(r1, r2): - return r1.path == r2.path + assert r1.path == r2.path, "{} != {}".format(r1.path, r2.path) def query(r1, r2): - return r1.query == r2.query + assert r1.query == r2.query, "{} != {}".format(r1.query, r2.query) def raw_body(r1, r2): - return read_body(r1) == read_body(r2) + assert read_body(r1) == read_body(r2) + + +def body(r1, r2): + transformer = _get_transformer(r1) + r2_transformer = _get_transformer(r2) + if transformer != r2_transformer: + transformer = _identity + assert transformer(read_body(r1)) == transformer(read_body(r2)) + + +def headers(r1, r2): + assert r1.headers == r2.headers, "{} != {}".format(r1.headers, r2.headers) def _header_checker(value, header='Content-Type'): @@ -74,28 +86,67 @@ def _get_transformer(request): return _identity -def body(r1, r2): - transformer = _get_transformer(r1) - r2_transformer = _get_transformer(r2) - if transformer != r2_transformer: - transformer = _identity - return transformer(read_body(r1)) == transformer(read_body(r2)) - - -def headers(r1, r2): - return r1.headers == r2.headers - - -def _log_matches(r1, r2, matches): - differences = [m for m in matches if not m[0]] - if differences: - log.debug( - "Requests {} and {} differ according to " - "the following matchers: {}".format(r1, r2, differences) - ) - - def requests_match(r1, r2, matchers): - matches = [(m(r1, r2), m) for m in matchers] - _log_matches(r1, r2, matches) - return all(m[0] for m in matches) + successes, failures = get_matchers_results(r1, r2, matchers) + if failures: + log.debug( + "Requests {} and {} differ.\n" + "Failure details:\n" + "{}".format(r1, r2, failures) + ) + return len(failures) == 0 + + +def _evaluate_matcher(matcher_function, *args): + """ + Evaluate the result of a given matcher as a boolean with an assertion error message if any. + It handles two types of matcher : + - a matcher returning a boolean value. + - a matcher that only makes an assert, returning None or raises an assertion error. + """ + assertion_message = None + try: + match = matcher_function(*args) + match = True if match is None else match + except AssertionError as e: + match = False + assertion_message = str(e) + return match, assertion_message + + +def get_matchers_results(r1, r2, matchers): + """ + Get the comparison results of two requests as two list. + The first returned list represents the matchers names that passed. + The second list is the failed matchers as a string with failed assertion details if any. + """ + matches_success, matches_fails = [], [] + for m in matchers: + matcher_name = m.__name__ + match, assertion_message = _evaluate_matcher(m, r1, r2) + if match: + matches_success.append(matcher_name) + else: + assertion_message = get_assertion_message(assertion_message) + matches_fails.append((matcher_name, assertion_message)) + return matches_success, matches_fails + + +def get_assertion_message(assertion_details, **format_options): + """ + Get a detailed message about the failing matcher. + """ + msg = "" + if assertion_details: + separator = format_options.get("separator", "-") + title = format_options.get("title", " DETAILS ") + nb_separator = format_options.get("nb_separator", 40) + first_title_line = ( + separator * ((nb_separator - len(title)) // 2) + + title + + separator * ((nb_separator - len(title)) // 2) + ) + msg += "{}\n{}\n{}\n".format( + first_title_line, str(assertion_details), separator * nb_separator + ) + return msg diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index c01ba59..05490bb 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -230,11 +230,8 @@ class VCRConnection(object): self._vcr_request ): raise CannotOverwriteExistingCassetteException( - "No match for the request (%r) was found. " - "Can't overwrite existing cassette (%r) in " - "your current record mode (%r)." - % (self._vcr_request, self.cassette._path, - self.cassette.record_mode) + cassette=self.cassette, + failed_request=self._vcr_request ) # Otherwise, we should send the request, then get the response diff --git a/vcr/stubs/tornado_stubs.py b/vcr/stubs/tornado_stubs.py index b675065..5beea4b 100644 --- a/vcr/stubs/tornado_stubs.py +++ b/vcr/stubs/tornado_stubs.py @@ -75,10 +75,8 @@ def vcr_fetch_impl(cassette, real_fetch_impl): request, 599, error=CannotOverwriteExistingCassetteException( - "No match for the request (%r) was found. " - "Can't overwrite existing cassette (%r) in " - "your current record mode (%r)." - % (vcr_request, cassette._path, cassette.record_mode) + cassette=cassette, + failed_request=vcr_request ), request_time=self.io_loop.time() - request.start_time, )