Merge pull request #439 from arthurHamon2/master

Add verbosity/explanations on CannotOverwriteExistingCassetteException

docs/advanced.rst

@@ -97,8 +97,12 @@ Create your own method with the following signature
 
     def my_matcher(r1, r2):
 
-Your method receives the two requests and must return ``True`` if they
-match, ``False`` if they don't.
+Your method receives the two requests and can return :
+
+- Use an ``assert`` statement in the matcher, then we have ``None`` if they match, raise an `AssertionError`` if they don't.
+- A boolean, ``True`` if they match, ``False`` if they don't.
+
+Note : You should use an ``assert`` statement in order to have feedback when a matcher is failing.
 
 Finally, register your method with VCR to use your new request matcher.
 
@@ -107,7 +111,7 @@ Finally, register your method with VCR to use your new request matcher.
     import vcr
 
     def jurassic_matcher(r1, r2):
-        return r1.uri == r2.uri and 'JURASSIC PARK' in r1.body
+        assert r1.uri == r2.uri and 'JURASSIC PARK' in r1.body
 
     my_vcr = vcr.VCR()
     my_vcr.register_matcher('jurassic', jurassic_matcher)

docs/usage.rst

@@ -95,3 +95,9 @@ Unittest Integration
 While it's possible to use the context manager or decorator forms with unittest,
 there's also a ``VCRTestCase`` provided separately by `vcrpy-unittest
 <https://github.com/agriffis/vcrpy-unittest>`__.
+
+Pytest Integration
+------------------
+
+A Pytest plugin is available here : `pytest-vcr
+<https://github.com/ktosiek/pytest-vcr>`__.

tests/unit/test_cassettes.py

@@ -317,3 +317,51 @@ def test_use_as_decorator_on_generator():
         yield 2
 
     assert list(test_function()) == [1, 2]
+
+
+@mock.patch("vcr.cassette.get_matchers_results")
+def test_find_requests_with_most_matches_one_similar_request(mock_get_matchers_results):
+    mock_get_matchers_results.side_effect = [
+        (["method"], [("path", "failed : path"), ("query", "failed : query")]),
+        (["method", "path"], [("query", "failed : query")]),
+        ([], [("method", "failed : method"), ("path", "failed : path"), ("query", "failed : query")]),
+    ]
+
+    cassette = Cassette("test")
+    for request in range(1, 4):
+        cassette.append(request, 'response')
+    result = cassette.find_requests_with_most_matches("fake request")
+    assert result == [(2, ["method", "path"], [("query", "failed : query")])]
+
+
+@mock.patch("vcr.cassette.get_matchers_results")
+def test_find_requests_with_most_matches_no_similar_requests(mock_get_matchers_results):
+    mock_get_matchers_results.side_effect = [
+        ([], [("path", "failed : path"), ("query", "failed : query")]),
+        ([], [("path", "failed : path"), ("query", "failed : query")]),
+        ([], [("path", "failed : path"), ("query", "failed : query")]),
+    ]
+
+    cassette = Cassette("test")
+    for request in range(1, 4):
+        cassette.append(request, 'response')
+    result = cassette.find_requests_with_most_matches("fake request")
+    assert result == []
+
+
+@mock.patch("vcr.cassette.get_matchers_results")
+def test_find_requests_with_most_matches_many_similar_requests(mock_get_matchers_results):
+    mock_get_matchers_results.side_effect = [
+        (["method", "path"], [("query", "failed : query")]),
+        (["method"], [("path", "failed : path"), ("query", "failed : query")]),
+        (["method", "path"], [("query", "failed : query")]),
+    ]
+
+    cassette = Cassette("test")
+    for request in range(1, 4):
+        cassette.append(request, 'response')
+    result = cassette.find_requests_with_most_matches("fake request")
+    assert result == [
+        (1, ["method", "path"], [("query", "failed : query")]),
+        (3, ["method", "path"], [("query", "failed : query")])
+    ]

tests/unit/test_matchers.py

@@ -1,4 +1,5 @@
 import itertools
+from vcr.compat import mock
 
 import pytest
 
@@ -21,20 +22,22 @@ REQUESTS = {
 def assert_matcher(matcher_name):
     matcher = getattr(matchers, matcher_name)
     for k1, k2 in itertools.permutations(REQUESTS, 2):
-        matched = matcher(REQUESTS[k1], REQUESTS[k2])
-        if matcher_name in {k1, k2}:
-            assert not matched
+        expecting_assertion_error = matcher_name in {k1, k2}
+        if expecting_assertion_error:
+            with pytest.raises(AssertionError):
+                matcher(REQUESTS[k1], REQUESTS[k2])
         else:
-            assert matched
+            assert matcher(REQUESTS[k1], REQUESTS[k2]) is None
 
 
 def test_uri_matcher():
     for k1, k2 in itertools.permutations(REQUESTS, 2):
-        matched = matchers.uri(REQUESTS[k1], REQUESTS[k2])
-        if {k1, k2} != {'base', 'method'}:
-            assert not matched
+        expecting_assertion_error = {k1, k2} != {"base", "method"}
+        if expecting_assertion_error:
+            with pytest.raises(AssertionError):
+                matchers.uri(REQUESTS[k1], REQUESTS[k2])
         else:
-            assert matched
+            assert matchers.uri(REQUESTS[k1], REQUESTS[k2]) is None
 
 
 req1_body = (b"<?xml version='1.0'?><methodCall><methodName>test</methodName>"
@@ -107,7 +110,7 @@ req2_body = (b"<?xml version='1.0'?><methodCall><methodName>test</methodName>"
     )
 ])
 def test_body_matcher_does_match(r1, r2):
-    assert matchers.body(r1, r2)
+    assert matchers.body(r1, r2) is None
 
 
 @pytest.mark.parametrize("r1, r2", [
@@ -135,25 +138,132 @@ def test_body_matcher_does_match(r1, r2):
     )
 ])
 def test_body_match_does_not_match(r1, r2):
-    assert not matchers.body(r1, r2)
+    with pytest.raises(AssertionError):
+        matchers.body(r1, r2)
 
 
 def test_query_matcher():
-    req1 = request.Request('GET', 'http://host.com/?a=b&c=d', '', {})
-    req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {})
-    assert matchers.query(req1, req2)
+    req1 = request.Request("GET", "http://host.com/?a=b&c=d", "", {})
+    req2 = request.Request("GET", "http://host.com/?c=d&a=b", "", {})
+    assert matchers.query(req1, req2) is None
 
-    req1 = request.Request('GET', 'http://host.com/?a=b&a=b&c=d', '', {})
-    req2 = request.Request('GET', 'http://host.com/?a=b&c=d&a=b', '', {})
-    req3 = request.Request('GET', 'http://host.com/?c=d&a=b&a=b', '', {})
-    assert matchers.query(req1, req2)
-    assert matchers.query(req1, req3)
+    req1 = request.Request("GET", "http://host.com/?a=b&a=b&c=d", "", {})
+    req2 = request.Request("GET", "http://host.com/?a=b&c=d&a=b", "", {})
+    req3 = request.Request("GET", "http://host.com/?c=d&a=b&a=b", "", {})
+    assert matchers.query(req1, req2) is None
+    assert matchers.query(req1, req3) is None
 
 
-def test_metchers():
-    assert_matcher('method')
-    assert_matcher('scheme')
-    assert_matcher('host')
-    assert_matcher('port')
-    assert_matcher('path')
-    assert_matcher('query')
+def test_matchers():
+    assert_matcher("method")
+    assert_matcher("scheme")
+    assert_matcher("host")
+    assert_matcher("port")
+    assert_matcher("path")
+    assert_matcher("query")
+
+
+def test_evaluate_matcher_does_match():
+    def bool_matcher(r1, r2):
+        return True
+
+    def assertion_matcher(r1, r2):
+        assert 1 == 1
+
+    r1, r2 = None, None
+    for matcher in [bool_matcher, assertion_matcher]:
+        match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2)
+        assert match is True
+        assert assertion_msg is None
+
+
+def test_evaluate_matcher_does_not_match():
+    def bool_matcher(r1, r2):
+        return False
+
+    def assertion_matcher(r1, r2):
+        # This is like the "assert" statement preventing pytest to recompile it
+        raise AssertionError()
+
+    r1, r2 = None, None
+    for matcher in [bool_matcher, assertion_matcher]:
+        match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2)
+        assert match is False
+        assert not assertion_msg
+
+
+def test_evaluate_matcher_does_not_match_with_assert_message():
+    def assertion_matcher(r1, r2):
+        # This is like the "assert" statement preventing pytest to recompile it
+        raise AssertionError("Failing matcher")
+
+    r1, r2 = None, None
+    match, assertion_msg = matchers._evaluate_matcher(assertion_matcher, r1, r2)
+    assert match is False
+    assert assertion_msg == "Failing matcher"
+
+
+def test_get_assertion_message():
+    assert matchers.get_assertion_message(None) == ""
+    assert matchers.get_assertion_message("") == ""
+
+
+def test_get_assertion_message_with_details():
+    assertion_msg = "q1=1 != q2=1"
+    expected = (
+        "--------------- DETAILS ---------------\n"
+        "{}\n"
+        "----------------------------------------\n".format(assertion_msg)
+    )
+    assert matchers.get_assertion_message(assertion_msg) == expected
+
+
+@pytest.mark.parametrize(
+    "r1, r2, expected_successes, expected_failures",
+    [
+        (
+            request.Request("GET", "http://host.com/p?a=b", "", {}),
+            request.Request("GET", "http://host.com/p?a=b", "", {}),
+            ["method", "path"],
+            [],
+        ),
+        (
+            request.Request("GET", "http://host.com/p?a=b", "", {}),
+            request.Request("POST", "http://host.com/p?a=b", "", {}),
+            ["path"],
+            ["method"],
+        ),
+        (
+            request.Request("GET", "http://host.com/p?a=b", "", {}),
+            request.Request("POST", "http://host.com/path?a=b", "", {}),
+            [],
+            ["method", "path"],
+        ),
+    ],
+)
+def test_get_matchers_results(r1, r2, expected_successes, expected_failures):
+    successes, failures = matchers.get_matchers_results(
+        r1, r2, [matchers.method, matchers.path]
+    )
+    assert successes == expected_successes
+    assert len(failures) == len(expected_failures)
+    for i, expected_failure in enumerate(expected_failures):
+        assert failures[i][0] == expected_failure
+        assert failures[i][1] is not None
+
+
+@mock.patch("vcr.matchers.get_matchers_results")
+@pytest.mark.parametrize(
+    "successes, failures, expected_match",
+    [
+        (["method", "path"], [], True),
+        (["method"], ["path"], False),
+        ([], ["method", "path"], False),
+    ],
+)
+def test_requests_match(mock_get_matchers_results, successes, failures, expected_match):
+    mock_get_matchers_results.return_value = (successes, failures)
+    r1 = request.Request("GET", "http://host.com/p?a=b", "", {})
+    r2 = request.Request("GET", "http://host.com/p?a=b", "", {})
+    match = matchers.requests_match(r1, r2, [matchers.method, matchers.path])
+    assert match is expected_match

vcr/cassette.py

@@ -8,7 +8,7 @@ import wrapt
 
 from .compat import contextlib
 from .errors import UnhandledHTTPRequestError
-from .matchers import requests_match, uri, method
+from .matchers import requests_match, uri, method, get_matchers_results
 from .patch import CassettePatcherBuilder
 from .serializers import yamlserializer
 from .persisters.filesystem import FilesystemPersister
@@ -290,6 +290,38 @@ class Cassette(object):
     def rewind(self):
         self.play_counts = collections.Counter()
 
+    def find_requests_with_most_matches(self, request):
+        """
+        Get the most similar request(s) stored in the cassette
+        of a given request as a list of tuples like this:
+        - the request object
+        - the successful matchers as string
+        - the failed matchers and the related assertion message with the difference details as strings tuple
+
+        This is useful when a request failed to be found,
+        we can get the similar request(s) in order to know what have changed in the request parts.
+        """
+        best_matches = []
+        request = self._before_record_request(request)
+        for index, (stored_request, response) in enumerate(self.data):
+            successes, fails = get_matchers_results(request, stored_request, self._match_on)
+            best_matches.append((len(successes), stored_request, successes, fails))
+        best_matches.sort(key=lambda t: t[0], reverse=True)
+        # Get the first best matches (multiple if equal matches)
+        final_best_matches = []
+        previous_nb_success = best_matches[0][0]
+        for best_match in best_matches:
+            nb_success = best_match[0]
+            # Do not keep matches that have 0 successes,
+            # it means that the request is totally different from
+            # the ones stored in the cassette
+            if nb_success < 1 or previous_nb_success != nb_success:
+                break
+            previous_nb_success = nb_success
+            final_best_matches.append(best_match[1:])
+
+        return final_best_matches
+
     def _as_dict(self):
         return {"requests": self.requests, "responses": self.responses}
 

vcr/errors.py

@@ -1,5 +1,30 @@
 class CannotOverwriteExistingCassetteException(Exception):
-    pass
+    def __init__(self, *args, **kwargs):
+        message = self._get_message(kwargs["cassette"], kwargs["failed_request"])
+        super(CannotOverwriteExistingCassetteException, self).__init__(message)
+
+    def _get_message(self, cassette, failed_request):
+        """Get the final message related to the exception"""
+        # Get the similar requests in the cassette that
+        # have match the most with the request.
+        best_matches = cassette.find_requests_with_most_matches(failed_request)
+        # Build a comprehensible message to put in the exception.
+        best_matches_msg = ""
+        for best_match in best_matches:
+            request, _, failed_matchers_assertion_msgs = best_match
+            best_matches_msg += "Similar request found : (%r).\n" % request
+            for failed_matcher, assertion_msg in failed_matchers_assertion_msgs:
+                best_matches_msg += "Matcher failed : %s\n" "%s\n" % (
+                    failed_matcher,
+                    assertion_msg,
+                )
+        return (
+            "Can't overwrite existing cassette (%r) in "
+            "your current record mode (%r).\n"
+            "No match for the request (%r) was found.\n"
+            "%s"
+            % (cassette._path, cassette.record_mode, failed_request, best_matches_msg)
+        )
 
 
 class UnhandledHTTPRequestError(KeyError):

vcr/matchers.py

@@ -8,35 +8,47 @@ log = logging.getLogger(__name__)
 
 
 def method(r1, r2):
-    return r1.method == r2.method
+    assert r1.method == r2.method, "{} != {}".format(r1.method, r2.method)
 
 
 def uri(r1, r2):
-    return r1.uri == r2.uri
+    assert r1.uri == r2.uri, "{} != {}".format(r1.uri, r2.uri)
 
 
 def host(r1, r2):
-    return r1.host == r2.host
+    assert r1.host == r2.host, "{} != {}".format(r1.host, r2.host)
 
 
 def scheme(r1, r2):
-    return r1.scheme == r2.scheme
+    assert r1.scheme == r2.scheme, "{} != {}".format(r1.scheme, r2.scheme)
 
 
 def port(r1, r2):
-    return r1.port == r2.port
+    assert r1.port == r2.port, "{} != {}".format(r1.port, r2.port)
 
 
 def path(r1, r2):
-    return r1.path == r2.path
+    assert r1.path == r2.path, "{} != {}".format(r1.path, r2.path)
 
 
 def query(r1, r2):
-    return r1.query == r2.query
+    assert r1.query == r2.query, "{} != {}".format(r1.query, r2.query)
 
 
 def raw_body(r1, r2):
-    return read_body(r1) == read_body(r2)
+    assert read_body(r1) == read_body(r2)
+
+
+def body(r1, r2):
+    transformer = _get_transformer(r1)
+    r2_transformer = _get_transformer(r2)
+    if transformer != r2_transformer:
+        transformer = _identity
+    assert transformer(read_body(r1)) == transformer(read_body(r2))
+
+
+def headers(r1, r2):
+    assert r1.headers == r2.headers, "{} != {}".format(r1.headers, r2.headers)
 
 
 def _header_checker(value, header='Content-Type'):
@@ -74,28 +86,67 @@ def _get_transformer(request):
         return _identity
 
 
-def body(r1, r2):
-    transformer = _get_transformer(r1)
-    r2_transformer = _get_transformer(r2)
-    if transformer != r2_transformer:
-        transformer = _identity
-    return transformer(read_body(r1)) == transformer(read_body(r2))
-
-
-def headers(r1, r2):
-    return r1.headers == r2.headers
-
-
-def _log_matches(r1, r2, matches):
-    differences = [m for m in matches if not m[0]]
-    if differences:
-        log.debug(
-            "Requests {} and {} differ according to "
-            "the following matchers: {}".format(r1, r2, differences)
-        )
-
-
 def requests_match(r1, r2, matchers):
-    matches = [(m(r1, r2), m) for m in matchers]
-    _log_matches(r1, r2, matches)
-    return all(m[0] for m in matches)
+    successes, failures = get_matchers_results(r1, r2, matchers)
+    if failures:
+        log.debug(
+            "Requests {} and {} differ.\n"
+            "Failure details:\n"
+            "{}".format(r1, r2, failures)
+        )
+    return len(failures) == 0
+
+
+def _evaluate_matcher(matcher_function, *args):
+    """
+    Evaluate the result of a given matcher as a boolean with an assertion error message if any.
+    It handles two types of matcher :
+    - a matcher returning a boolean value.
+    - a matcher that only makes an assert, returning None or raises an assertion error.
+    """
+    assertion_message = None
+    try:
+        match = matcher_function(*args)
+        match = True if match is None else match
+    except AssertionError as e:
+        match = False
+        assertion_message = str(e)
+    return match, assertion_message
+
+
+def get_matchers_results(r1, r2, matchers):
+    """
+    Get the comparison results of two requests as two list.
+    The first returned list represents the matchers names that passed.
+    The second list is the failed matchers as a string with failed assertion details if any.
+    """
+    matches_success, matches_fails = [], []
+    for m in matchers:
+        matcher_name = m.__name__
+        match, assertion_message = _evaluate_matcher(m, r1, r2)
+        if match:
+            matches_success.append(matcher_name)
+        else:
+            assertion_message = get_assertion_message(assertion_message)
+            matches_fails.append((matcher_name, assertion_message))
+    return matches_success, matches_fails
+
+
+def get_assertion_message(assertion_details, **format_options):
+    """
+    Get a detailed message about the failing matcher.
+    """
+    msg = ""
+    if assertion_details:
+        separator = format_options.get("separator", "-")
+        title = format_options.get("title", " DETAILS ")
+        nb_separator = format_options.get("nb_separator", 40)
+        first_title_line = (
+            separator * ((nb_separator - len(title)) // 2)
+            + title
+            + separator * ((nb_separator - len(title)) // 2)
+        )
+        msg += "{}\n{}\n{}\n".format(
+            first_title_line, str(assertion_details), separator * nb_separator
+        )
+    return msg

vcr/stubs/__init__.py

@@ -230,11 +230,8 @@ class VCRConnection(object):
                 self._vcr_request
             ):
                 raise CannotOverwriteExistingCassetteException(
-                    "No match for the request (%r) was found. "
-                    "Can't overwrite existing cassette (%r) in "
-                    "your current record mode (%r)."
-                    % (self._vcr_request, self.cassette._path,
-                       self.cassette.record_mode)
+                    cassette=self.cassette,
+                    failed_request=self._vcr_request
                 )
 
             # Otherwise, we should send the request, then get the response

vcr/stubs/tornado_stubs.py

@@ -75,10 +75,8 @@ def vcr_fetch_impl(cassette, real_fetch_impl):
                     request,
                     599,
                     error=CannotOverwriteExistingCassetteException(
-                        "No match for the request (%r) was found. "
-                        "Can't overwrite existing cassette (%r) in "
-                        "your current record mode (%r)."
-                        % (vcr_request, cassette._path, cassette.record_mode)
+                        cassette=cassette,
+                        failed_request=vcr_request
                     ),
                     request_time=self.io_loop.time() - request.start_time,
                 )