From 16c6135387b9d25e225c8a34e5df92ffa0203bf7 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 1 Apr 2014 00:09:24 +0200 Subject: [PATCH 01/58] Removed 'serializer' from name of test functions Because the name '_serializer_' has no relationships with this tests --- tests/integration/test_register_matcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_register_matcher.py b/tests/integration/test_register_matcher.py index 9e542be..ad1da1c 100644 --- a/tests/integration/test_register_matcher.py +++ b/tests/integration/test_register_matcher.py @@ -10,7 +10,7 @@ def false_matcher(r1, r2): return False -def test_registered_serializer_true_matcher(tmpdir): +def test_registered_true_matcher(tmpdir): my_vcr = vcr.VCR() my_vcr.register_matcher('true', true_matcher) testfile = str(tmpdir.join('test.yml')) @@ -25,7 +25,7 @@ def test_registered_serializer_true_matcher(tmpdir): urlopen('https://httpbin.org/get') -def test_registered_serializer_false_matcher(tmpdir): +def test_registered_false_matcher(tmpdir): my_vcr = vcr.VCR() my_vcr.register_matcher('false', false_matcher) testfile = str(tmpdir.join('test.yml')) From cd32f5114c66042849b7d7c8be54c2591d3d7a3a Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 1 Apr 2014 00:11:55 +0200 Subject: [PATCH 02/58] Added unit test for matcher 'method' --- tests/unit/test_matchers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/unit/test_matchers.py diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py new file mode 100644 index 0000000..23601b9 --- /dev/null +++ b/tests/unit/test_matchers.py @@ -0,0 +1,13 @@ +from vcr import matchers +from vcr import request + + +def test_method(): + req_get = request.Request('http', 'google.com', 80, 'GET', '/', '', {}) + assert True == matchers.method(req_get, req_get) + + req_get1 = request.Request('https', 'httpbin.org', 80, 'GET', '/', '', {}) + assert True == matchers.method(req_get, req_get1) + + req_post = request.Request('http', 'google.com', 80, 'POST', '/', '', {}) + assert False == matchers.method(req_get, req_post) From 792d6658932506c21849a67f6cca951eac8b83a1 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 1 Apr 2014 00:21:01 +0200 Subject: [PATCH 03/58] Added unit test for matcher 'url' --- tests/unit/test_matchers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 23601b9..0075485 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -11,3 +11,20 @@ def test_method(): req_post = request.Request('http', 'google.com', 80, 'POST', '/', '', {}) assert False == matchers.method(req_get, req_post) + + +def test_url(): + req1 = request.Request('http', 'google.com', 80, 'GET', '/', '', {}) + assert True == matchers.url(req1, req1) + + req2 = request.Request('http', 'httpbin.org', 80, 'GET', '/', '', {}) + assert False == matchers.url(req1, req2) + + req1_post = request.Request('http', 'google.com', 80, 'POST', '/', '', {}) + assert True == matchers.url(req1, req1_post) + + req_query_string = request.Request( + 'http', 'google.com?p1=t1&p2=t2', 80, 'GET', '/', '', {}) + req_query_string1 = request.Request( + 'http', 'google.com?p2=t2&p1=t1', 80, 'GET', '/', '', {}) + assert False == matchers.url(req_query_string, req_query_string1) From 08d4d8913ae80a31e927b3e243ad4dd057bb3b58 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 1 Apr 2014 22:44:03 +0200 Subject: [PATCH 04/58] Added integration test for match on 'method' --- tests/integration/test_matchers.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/integration/test_matchers.py diff --git a/tests/integration/test_matchers.py b/tests/integration/test_matchers.py new file mode 100644 index 0000000..1b7765d --- /dev/null +++ b/tests/integration/test_matchers.py @@ -0,0 +1,24 @@ +import vcr +import pytest +from six.moves.urllib.request import urlopen + + +@pytest.fixture +def cassette(tmpdir): + return str(tmpdir.join('test.yml')) + + +def test_method_matcher(cassette): + # prepare cassete + with vcr.use_cassette(cassette, match_on=['method']) as cass: + urlopen('http://httpbin.org/') + assert len(cass) == 1 + + # play cassette with matching on method + with vcr.use_cassette(cassette, match_on=['method']) as cass: + urlopen('http://httpbin.org/get') + assert cass.play_count == 1 + + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette, match_on=['method']) as cass: + urlopen('http://httpbin.org/post', data='') From e0c6a8429d6b81f7ffadad67f77501ab9a9d100a Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 1 Apr 2014 22:59:33 +0200 Subject: [PATCH 05/58] Added integration test for match on 'url' --- tests/integration/test_matchers.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_matchers.py b/tests/integration/test_matchers.py index 1b7765d..c4b7999 100644 --- a/tests/integration/test_matchers.py +++ b/tests/integration/test_matchers.py @@ -19,6 +19,24 @@ def test_method_matcher(cassette): urlopen('http://httpbin.org/get') assert cass.play_count == 1 + # should fail if method does not match with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): with vcr.use_cassette(cassette, match_on=['method']) as cass: - urlopen('http://httpbin.org/post', data='') + urlopen('http://httpbin.org/post', data=b'') # is a POST request + + +def test_url_matcher(cassette): + # prepare cassete + with vcr.use_cassette(cassette, match_on=['url']) as cass: + urlopen('http://httpbin.org/get?p1=q1&p2=q2') + assert len(cass) == 1 + + # play cassette with matching on url + with vcr.use_cassette(cassette, match_on=['url']) as cass: + urlopen('http://httpbin.org/get?p1=q1&p2=q2') + assert cass.play_count == 1 + + # should fail if url does not match + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette, match_on=['url']) as cass: + urlopen('http://httpbin.org/get?p2=q2&p1=q1') From edf1df9188ed69ce3ceb797a9598b9b28d02ab2c Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 6 Apr 2014 23:49:54 +0200 Subject: [PATCH 06/58] Replaced Request 'host, port, protocol, path' with 'uri' --- tests/fixtures/wild/domain_redirect.yaml | 10 ++------ tests/integration/test_request.py | 4 ++-- tests/unit/test_matchers.py | 16 ++++++------- tests/unit/test_request.py | 9 ++----- vcr/request.py | 20 +++++----------- vcr/stubs/__init__.py | 30 +++++++++++++++++------- 6 files changed, 41 insertions(+), 48 deletions(-) diff --git a/tests/fixtures/wild/domain_redirect.yaml b/tests/fixtures/wild/domain_redirect.yaml index 2b8af68..465f071 100644 --- a/tests/fixtures/wild/domain_redirect.yaml +++ b/tests/fixtures/wild/domain_redirect.yaml @@ -4,11 +4,8 @@ - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] - !!python/tuple [User-Agent, vcrpy-test] - !!python/tuple [Accept, '*/*'] - host: seomoz.org method: GET - path: / - port: 80 - protocol: http + uri: http://seomoz.org:80/ response: body: {string: ''} headers: ["Location: http://moz.com/\r\n", "Server: BigIP\r\n", "Connection: Keep-Alive\r\n", @@ -20,11 +17,8 @@ - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] - !!python/tuple [User-Agent, vcrpy-test] - !!python/tuple [Accept, '*/*'] - host: moz.com method: GET - path: / - port: 80 - protocol: http + uri: http://moz.com:80/ response: body: string: !!binary | diff --git a/tests/integration/test_request.py b/tests/integration/test_request.py index 829c6eb..882783e 100644 --- a/tests/integration/test_request.py +++ b/tests/integration/test_request.py @@ -6,6 +6,6 @@ def test_recorded_request_url_with_redirected_request(tmpdir): with vcr.use_cassette(str(tmpdir.join('test.yml'))) as cass: assert len(cass) == 0 urlopen('http://httpbin.org/redirect/3') - assert cass.requests[0].url == 'http://httpbin.org/redirect/3' - assert cass.requests[3].url == 'http://httpbin.org/get' + assert cass.requests[0].url == 'http://httpbin.org:80/redirect/3' + assert cass.requests[3].url == 'http://httpbin.org:80/get' assert len(cass) == 4 diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 0075485..056fef7 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -3,28 +3,28 @@ from vcr import request def test_method(): - req_get = request.Request('http', 'google.com', 80, 'GET', '/', '', {}) + req_get = request.Request('GET', 'http://google.com:80/', '', {}) assert True == matchers.method(req_get, req_get) - req_get1 = request.Request('https', 'httpbin.org', 80, 'GET', '/', '', {}) + req_get1 = request.Request('GET', 'https://httpbin.org:80/', '', {}) assert True == matchers.method(req_get, req_get1) - req_post = request.Request('http', 'google.com', 80, 'POST', '/', '', {}) + req_post = request.Request('POST', 'http://google.com:80/', '', {}) assert False == matchers.method(req_get, req_post) def test_url(): - req1 = request.Request('http', 'google.com', 80, 'GET', '/', '', {}) + req1 = request.Request('GET', 'http://google.com:80/', '', {}) assert True == matchers.url(req1, req1) - req2 = request.Request('http', 'httpbin.org', 80, 'GET', '/', '', {}) + req2 = request.Request('GET', 'https://httpbin.org:80/', '', {}) assert False == matchers.url(req1, req2) - req1_post = request.Request('http', 'google.com', 80, 'POST', '/', '', {}) + req1_post = request.Request('POST', 'http://google.com:80/', '', {}) assert True == matchers.url(req1, req1_post) req_query_string = request.Request( - 'http', 'google.com?p1=t1&p2=t2', 80, 'GET', '/', '', {}) + 'GET', 'http://google.com:80/?p1=t1&p2=t2', '', {}) req_query_string1 = request.Request( - 'http', 'google.com?p2=t2&p1=t1', 80, 'GET', '/', '', {}) + 'GET', 'http://google.com:80/?p2=t2&p1=t1', '', {}) assert False == matchers.url(req_query_string, req_query_string1) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index 304522a..adecee6 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -1,11 +1,6 @@ from vcr.request import Request -def test_url(): - req = Request('http', 'www.google.com', 80, 'GET', '/', '', {}) - assert req.url == 'http://www.google.com/' - - def test_str(): - req = Request('http', 'www.google.com', 80, 'GET', '/', '', {}) - str(req) == '' + req = Request('GET', 'http://www.google.com:80/', '', {}) + str(req) == '' diff --git a/vcr/request.py b/vcr/request.py index bbe950b..a224225 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -1,11 +1,8 @@ class Request(object): - def __init__(self, protocol, host, port, method, path, body, headers): - self.protocol = protocol - self.host = host - self.port = port + def __init__(self, method, uri, body, headers): self.method = method - self.path = path + self.uri = uri self.body = body # make headers a frozenset so it will be hashable self.headers = frozenset(headers.items()) @@ -17,14 +14,12 @@ class Request(object): @property def url(self): - return "{0}://{1}{2}".format(self.protocol, self.host, self.path) + return self.uri def __key(self): return ( - self.host, - self.port, self.method, - self.path, + self.uri, self.body, self.headers ) @@ -36,18 +31,15 @@ class Request(object): return hash(self) == hash(other) def __str__(self): - return "".format(self.method, self.url) + return "".format(self.method, self.uri) def __repr__(self): return self.__str__() def _to_dict(self): return { - 'protocol': self.protocol, - 'host': self.host, - 'port': self.port, 'method': self.method, - 'path': self.path, + 'uri': self.uri, 'body': self.body, 'headers': self.headers, } diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 04c0f79..77391b2 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -119,14 +119,29 @@ class VCRConnection: # A reference to the cassette that's currently being patched in cassette = None + def _uri(self, url): + """Returns request absolute URI""" + return "{0}://{1}:{2}{3}".format( + self._protocol, + self.real_connection.host, + self.real_connection.port, + url, + ) + + def _url(self, uri): + """Returns request selector url from absolute URI""" + prefix = "{0}://{1}:{2}".format( + self._protocol, + self.real_connection.host, + self.real_connection.port, + ) + return uri.replace(prefix, '', 1) + def request(self, method, url, body=None, headers=None): '''Persist the request metadata in self._vcr_request''' self._vcr_request = Request( - protocol=self._protocol, - host=self.real_connection.host, - port=self.real_connection.port, method=method, - path=url, + uri=self._uri(url), body=body, headers=headers or {} ) @@ -144,11 +159,8 @@ class VCRConnection: of putheader() calls. """ self._vcr_request = Request( - protocol=self._protocol, - host=self.real_connection.host, - port=self.real_connection.port, method=method, - path=url, + uri=self._uri(url), body="", headers={} ) @@ -211,7 +223,7 @@ class VCRConnection: ) self.real_connection.request( method=self._vcr_request.method, - url=self._vcr_request.path, + url=self._url(self._vcr_request.uri), body=self._vcr_request.body, headers=dict(self._vcr_request.headers or {}) ) From 6cca703eeef9f55a7c694b922cacdb2d872f705f Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 00:55:06 +0200 Subject: [PATCH 07/58] Refactored unit test for matchers --- tests/unit/test_matchers.py | 55 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 056fef7..3bc8846 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -1,30 +1,41 @@ +import itertools + from vcr import matchers from vcr import request - -def test_method(): - req_get = request.Request('GET', 'http://google.com:80/', '', {}) - assert True == matchers.method(req_get, req_get) - - req_get1 = request.Request('GET', 'https://httpbin.org:80/', '', {}) - assert True == matchers.method(req_get, req_get1) - - req_post = request.Request('POST', 'http://google.com:80/', '', {}) - assert False == matchers.method(req_get, req_post) +# the dict contains requests with corresponding to its key difference +# with 'base' request. +REQUESTS = { + 'base': request.Request('GET', 'http://host.com:80/', '', {}), + 'method': request.Request('POST', 'http://host.com:80/', '', {}), + 'protocol': request.Request('GET', 'https://host.com:80/', '', {}), + 'host': request.Request('GET', 'http://another-host.com:80/', '', {}), + 'port': request.Request('GET', 'http://host.com:90/', '', {}), + 'path': request.Request('GET', 'http://host.com:80/a', '', {}), + 'query': request.Request('GET', 'http://host.com:80/?a=b', '', {}), +} -def test_url(): - req1 = request.Request('GET', 'http://google.com:80/', '', {}) - assert True == matchers.url(req1, req1) +def assert_matcher(matcher_name): + matcher = getattr(matchers, matcher_name) + for k1, k2 in itertools.permutations(REQUESTS, 2): + matched = matcher(REQUESTS[k1], REQUESTS[k2]) + if matcher_name in {k1, k2}: + assert not matched + else: + assert matched - req2 = request.Request('GET', 'https://httpbin.org:80/', '', {}) - assert False == matchers.url(req1, req2) - req1_post = request.Request('POST', 'http://google.com:80/', '', {}) - assert True == matchers.url(req1, req1_post) +def test_url_matcher(): + for k1, k2 in itertools.permutations(REQUESTS, 2): + matched = matchers.url(REQUESTS[k1], REQUESTS[k2]) + if {k1, k2} != {'base', 'method'}: + assert not matched + else: + assert matched - req_query_string = request.Request( - 'GET', 'http://google.com:80/?p1=t1&p2=t2', '', {}) - req_query_string1 = request.Request( - 'GET', 'http://google.com:80/?p2=t2&p1=t1', '', {}) - assert False == matchers.url(req_query_string, req_query_string1) + +def test_metchers(): + assert_matcher('method') + #assert_matcher('method') + #assert_matcher('method') From 18ec57fa73436635eeabd69008c7bd5cfa5e6e13 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:08:57 +0200 Subject: [PATCH 08/58] Added test and impl for Request 'host' attribute --- tests/unit/test_matchers.py | 1 + vcr/request.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 3bc8846..f06db11 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -37,5 +37,6 @@ def test_url_matcher(): def test_metchers(): assert_matcher('method') + assert_matcher('host') #assert_matcher('method') #assert_matcher('method') diff --git a/vcr/request.py b/vcr/request.py index a224225..7debabf 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -1,3 +1,6 @@ +from six.moves.urllib.parse import urlparse, parse_qsl + + class Request(object): def __init__(self, method, uri, body, headers): @@ -12,6 +15,10 @@ class Request(object): tmp[key] = value self.headers = frozenset(tmp.iteritems()) + @property + def host(self): + return urlparse(self.uri).hostname + @property def url(self): return self.uri From bd9fa773e8d4ceca405028043e22eb0d6ccee0bd Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:12:05 +0200 Subject: [PATCH 09/58] Added port to Request with matcher and test --- tests/unit/test_matchers.py | 2 +- vcr/matchers.py | 4 ++++ vcr/request.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index f06db11..d6e4123 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -38,5 +38,5 @@ def test_url_matcher(): def test_metchers(): assert_matcher('method') assert_matcher('host') - #assert_matcher('method') + assert_matcher('port') #assert_matcher('method') diff --git a/vcr/matchers.py b/vcr/matchers.py index 0adf3a8..1bdc515 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -13,6 +13,10 @@ def host(r1, r2): return r1.host == r2.host +def port(r1, r2): + return r1.port == r2.port + + def path(r1, r2): return r1.path == r2.path diff --git a/vcr/request.py b/vcr/request.py index 7debabf..8b45a47 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -19,6 +19,10 @@ class Request(object): def host(self): return urlparse(self.uri).hostname + @property + def port(self): + return urlparse(self.uri).port + @property def url(self): return self.uri From 6b060e56667940176c234fb3fdc6ce8c6a02d5a1 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:13:51 +0200 Subject: [PATCH 10/58] Added path to Request with matcher and test --- tests/unit/test_matchers.py | 2 +- vcr/request.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index d6e4123..3e69456 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -39,4 +39,4 @@ def test_metchers(): assert_matcher('method') assert_matcher('host') assert_matcher('port') - #assert_matcher('method') + assert_matcher('path') diff --git a/vcr/request.py b/vcr/request.py index 8b45a47..3cae183 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -23,6 +23,10 @@ class Request(object): def port(self): return urlparse(self.uri).port + @property + def path(self): + return urlparse(self.uri).path + @property def url(self): return self.uri From 9b188e986f1ce5d5eb5572ce395623f94001f51f Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:30:31 +0200 Subject: [PATCH 11/58] Added query to Request with matcher and test --- tests/unit/test_matchers.py | 13 +++++++++++++ vcr/matchers.py | 4 ++++ vcr/request.py | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 3e69456..2462fce 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -35,8 +35,21 @@ def test_url_matcher(): assert matched +def test_query_matcher(): + req1 = request.Request('GET', 'http://host.com:80/?a=b&c=d', '', {}) + req2 = request.Request('GET', 'http://host.com:80/?c=d&a=b', '', {}) + assert matchers.query(req1, req2) + + req1 = request.Request('GET', 'http://host.com:80/?a=b&a=b&c=d', '', {}) + req2 = request.Request('GET', 'http://host.com:80/?a=b&c=d&a=b', '', {}) + req3 = request.Request('GET', 'http://host.com:80/?c=d&a=b&a=b', '', {}) + assert matchers.query(req1, req2) + assert matchers.query(req1, req3) + + def test_metchers(): assert_matcher('method') assert_matcher('host') assert_matcher('port') assert_matcher('path') + assert_matcher('query') diff --git a/vcr/matchers.py b/vcr/matchers.py index 1bdc515..4dde6a0 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -21,6 +21,10 @@ def path(r1, r2): return r1.path == r2.path +def query(r1, r2): + return r1.query == r2.query + + def body(r1, r2): return r1.body == r2.body diff --git a/vcr/request.py b/vcr/request.py index 3cae183..ca6faee 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -27,6 +27,11 @@ class Request(object): def path(self): return urlparse(self.uri).path + @property + def query(self): + q = urlparse(self.uri).query + return sorted(parse_qsl(q)) + @property def url(self): return self.uri From 5015dbd8785f2fd2822aa0f6f9331fb5b7cf4a0e Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:30:48 +0200 Subject: [PATCH 12/58] Improved test samples --- tests/unit/test_matchers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 2462fce..e3c85fb 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -6,13 +6,13 @@ from vcr import request # the dict contains requests with corresponding to its key difference # with 'base' request. REQUESTS = { - 'base': request.Request('GET', 'http://host.com:80/', '', {}), - 'method': request.Request('POST', 'http://host.com:80/', '', {}), - 'protocol': request.Request('GET', 'https://host.com:80/', '', {}), - 'host': request.Request('GET', 'http://another-host.com:80/', '', {}), - 'port': request.Request('GET', 'http://host.com:90/', '', {}), - 'path': request.Request('GET', 'http://host.com:80/a', '', {}), - 'query': request.Request('GET', 'http://host.com:80/?a=b', '', {}), + 'base': request.Request('GET', 'http://host.com:80/p?a=b', '', {}), + 'method': request.Request('POST', 'http://host.com:80/p?a=b', '', {}), + 'protocol': request.Request('GET', 'https://host.com:80/p?a=b', '', {}), + 'host': request.Request('GET', 'http://another-host.com:80/p?a=b', '', {}), + 'port': request.Request('GET', 'http://host.com:90/p?a=b', '', {}), + 'path': request.Request('GET', 'http://host.com:80/x?a=b', '', {}), + 'query': request.Request('GET', 'http://host.com:80/p?c=d', '', {}), } From 4e9d5f68853f3a33881ff76b42a0e588f2a3a728 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:35:52 +0200 Subject: [PATCH 13/58] Updated default 'match_on' --- vcr/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vcr/config.py b/vcr/config.py index 4cc335f..657f706 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -9,7 +9,7 @@ class VCR(object): serializer='yaml', cassette_library_dir=None, record_mode="once", - match_on=['url', 'method'], + match_on=['method', 'host', 'port', 'path', 'query'], filter_headers=[], filter_query_parameters=[], before_record=None, @@ -25,7 +25,9 @@ class VCR(object): 'method': method, 'url': url, 'host': host, + 'port': method, 'path': path, + 'query': path, 'headers': headers, 'body': body, } From 4267828a3e45751902ee9ef3ba9cfd971ea15223 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:56:55 +0200 Subject: [PATCH 14/58] Added 'scheme' to Request with matcher and test --- tests/unit/test_matchers.py | 3 ++- vcr/matchers.py | 4 ++++ vcr/request.py | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index e3c85fb..30ab5ff 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -8,7 +8,7 @@ from vcr import request REQUESTS = { 'base': request.Request('GET', 'http://host.com:80/p?a=b', '', {}), 'method': request.Request('POST', 'http://host.com:80/p?a=b', '', {}), - 'protocol': request.Request('GET', 'https://host.com:80/p?a=b', '', {}), + 'scheme': request.Request('GET', 'https://host.com:80/p?a=b', '', {}), 'host': request.Request('GET', 'http://another-host.com:80/p?a=b', '', {}), 'port': request.Request('GET', 'http://host.com:90/p?a=b', '', {}), 'path': request.Request('GET', 'http://host.com:80/x?a=b', '', {}), @@ -49,6 +49,7 @@ def test_query_matcher(): def test_metchers(): assert_matcher('method') + assert_matcher('scheme') assert_matcher('host') assert_matcher('port') assert_matcher('path') diff --git a/vcr/matchers.py b/vcr/matchers.py index 4dde6a0..1918652 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -13,6 +13,10 @@ def host(r1, r2): return r1.host == r2.host +def scheme(r1, r2): + return r1.scheme == r2.scheme + + def port(r1, r2): return r1.port == r2.port diff --git a/vcr/request.py b/vcr/request.py index ca6faee..9e76352 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -15,6 +15,10 @@ class Request(object): tmp[key] = value self.headers = frozenset(tmp.iteritems()) + @property + def scheme(self): + return urlparse(self.uri).scheme + @property def host(self): return urlparse(self.uri).hostname From 2f6db0dc0c0b913c2b60c2039f0ced0abf1b04fc Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 01:57:35 +0200 Subject: [PATCH 15/58] Added scheme to default 'match_on' --- vcr/config.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/vcr/config.py b/vcr/config.py index 657f706..8feb0c2 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -1,7 +1,7 @@ import os from .cassette import Cassette from .serializers import yamlserializer, jsonserializer -from .matchers import method, url, host, path, headers, body +from . import matchers class VCR(object): @@ -9,10 +9,17 @@ class VCR(object): serializer='yaml', cassette_library_dir=None, record_mode="once", - match_on=['method', 'host', 'port', 'path', 'query'], filter_headers=[], filter_query_parameters=[], before_record=None, + match_on=[ + 'method', + 'scheme', + 'host', + 'port', + 'path', + 'query', + ], ): self.serializer = serializer self.match_on = match_on @@ -22,14 +29,15 @@ class VCR(object): 'json': jsonserializer, } self.matchers = { - 'method': method, - 'url': url, - 'host': host, - 'port': method, - 'path': path, - 'query': path, - 'headers': headers, - 'body': body, + 'method': matchers.method, + 'url': matchers.url, + 'scheme': matchers.scheme, + 'host': matchers.host, + 'port': matchers.method, + 'path': matchers.path, + 'query': matchers.path, + 'headers': matchers.headers, + 'body': matchers.body, } self.record_mode = record_mode self.filter_headers = filter_headers From 7fe55ad8b86b0df3ba7fa46ed547b0b2d52304f8 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 7 Apr 2014 02:03:32 +0200 Subject: [PATCH 16/58] Updated 'set' to be compatible with 2.6 --- tests/unit/test_matchers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 30ab5ff..ae29c8a 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -20,7 +20,7 @@ def assert_matcher(matcher_name): matcher = getattr(matchers, matcher_name) for k1, k2 in itertools.permutations(REQUESTS, 2): matched = matcher(REQUESTS[k1], REQUESTS[k2]) - if matcher_name in {k1, k2}: + if matcher_name in set((k1, k2)): assert not matched else: assert matched @@ -29,7 +29,7 @@ def assert_matcher(matcher_name): def test_url_matcher(): for k1, k2 in itertools.permutations(REQUESTS, 2): matched = matchers.url(REQUESTS[k1], REQUESTS[k2]) - if {k1, k2} != {'base', 'method'}: + if set((k1, k2)) != set(('base', 'method')): assert not matched else: assert matched From 2fa1aaa1f717719afc4792445751b53efaa2889d Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Fri, 11 Apr 2014 01:32:10 +0200 Subject: [PATCH 17/58] Replaced 'url' mather with 'uri'. --- README.md | 2 +- tests/integration/test_matchers.py | 12 ++++++------ tests/unit/test_matchers.py | 4 ++-- vcr/cassette.py | 4 ++-- vcr/config.py | 3 ++- vcr/matchers.py | 4 ++-- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 26b85b5..638db4d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ my_vcr = vcr.VCR( serializer = 'json', cassette_library_dir = 'fixtures/cassettes', record_mode = 'once', - match_on = ['url', 'method'], + match_on = ['uri', 'method'], ) with my_vcr.use_cassette('test.json'): diff --git a/tests/integration/test_matchers.py b/tests/integration/test_matchers.py index c4b7999..007a844 100644 --- a/tests/integration/test_matchers.py +++ b/tests/integration/test_matchers.py @@ -25,18 +25,18 @@ def test_method_matcher(cassette): urlopen('http://httpbin.org/post', data=b'') # is a POST request -def test_url_matcher(cassette): +def test_uri_matcher(cassette): # prepare cassete - with vcr.use_cassette(cassette, match_on=['url']) as cass: + with vcr.use_cassette(cassette, match_on=['uri']) as cass: urlopen('http://httpbin.org/get?p1=q1&p2=q2') assert len(cass) == 1 - # play cassette with matching on url - with vcr.use_cassette(cassette, match_on=['url']) as cass: + # play cassette with matching on uri + with vcr.use_cassette(cassette, match_on=['uri']) as cass: urlopen('http://httpbin.org/get?p1=q1&p2=q2') assert cass.play_count == 1 - # should fail if url does not match + # should fail if uri does not match with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): - with vcr.use_cassette(cassette, match_on=['url']) as cass: + with vcr.use_cassette(cassette, match_on=['uri']) as cass: urlopen('http://httpbin.org/get?p2=q2&p1=q1') diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index ae29c8a..0d53fd7 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -26,9 +26,9 @@ def assert_matcher(matcher_name): assert matched -def test_url_matcher(): +def test_uri_matcher(): for k1, k2 in itertools.permutations(REQUESTS, 2): - matched = matchers.url(REQUESTS[k1], REQUESTS[k2]) + matched = matchers.uri(REQUESTS[k1], REQUESTS[k2]) if set((k1, k2)) != set(('base', 'method')): assert not matched else: diff --git a/vcr/cassette.py b/vcr/cassette.py index df6a64b..f71bf03 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -13,7 +13,7 @@ from .patch import install, reset from .persist import load_cassette, save_cassette from .filters import filter_request from .serializers import yamlserializer -from .matchers import requests_match, url, method +from .matchers import requests_match, uri, method from .errors import UnhandledHTTPRequestError @@ -31,7 +31,7 @@ class Cassette(ContextDecorator): path, serializer=yamlserializer, record_mode='once', - match_on=[url, method], + match_on=[uri, method]): filter_headers=[], filter_query_parameters=[], before_record=None, diff --git a/vcr/config.py b/vcr/config.py index 8feb0c2..e8848ff 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -30,7 +30,8 @@ class VCR(object): } self.matchers = { 'method': matchers.method, - 'url': matchers.url, + 'uri': matchers.uri, + 'url': matchers.uri, # matcher for backwards compatibility 'scheme': matchers.scheme, 'host': matchers.host, 'port': matchers.method, diff --git a/vcr/matchers.py b/vcr/matchers.py index 1918652..c471538 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -5,8 +5,8 @@ def method(r1, r2): return r1.method == r2.method -def url(r1, r2): - return r1.url == r2.url +def uri(r1, r2): + return r1.uri == r2.uri def host(r1, r2): From f9a64e1609e8e1cbf94c02127e605e136014899b Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Fri, 11 Apr 2014 01:33:30 +0200 Subject: [PATCH 18/58] Fixed available matchers declaration --- vcr/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vcr/config.py b/vcr/config.py index e8848ff..9789583 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -34,9 +34,9 @@ class VCR(object): 'url': matchers.uri, # matcher for backwards compatibility 'scheme': matchers.scheme, 'host': matchers.host, - 'port': matchers.method, + 'port': matchers.port, 'path': matchers.path, - 'query': matchers.path, + 'query': matchers.query, 'headers': matchers.headers, 'body': matchers.body, } From 96d8782d086dcfba1ff0563fd88f19db8630ef0b Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Fri, 11 Apr 2014 01:35:57 +0200 Subject: [PATCH 19/58] Added 'protocol' to Request for backwards compatibility --- vcr/request.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vcr/request.py b/vcr/request.py index 9e76352..d9dac9d 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -36,10 +36,16 @@ class Request(object): q = urlparse(self.uri).query return sorted(parse_qsl(q)) + # alias for backwards compatibility @property def url(self): return self.uri + # alias for backwards compatibility + @property + def protocol(self): + return self.scheme + def __key(self): return ( self.method, From a042cb38241cd2bba3451e2efd17f5ed31e0c467 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Fri, 11 Apr 2014 01:37:17 +0200 Subject: [PATCH 20/58] Updated README --- README.md | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 638db4d..e7106ae 100644 --- a/README.md +++ b/README.md @@ -92,20 +92,26 @@ Note: Per-cassette overrides take precedence over the global config. ## Request matching Request matching is configurable and allows you to change which requests VCR -considers identical. The default behavior is `['url', method']` which means -that requests with both the same URL and method (ie POST or GET) are considered +considers identical. The default behavior is +`['method', 'scheme', 'host', 'port', 'path', 'query']` which means that +requests with both the same URL and method (ie POST or GET) are considered identical. This can be configured by changing the `match_on` setting. The following options are available : - * method (for example, POST or GET) - * url (the full URL, including the protocol) - * host (the hostname of the server receiving the request) - * path (excluding the hostname) - * body (the entire request body) - * headers (the headers of the request) + * method (for example, POST or GET) + * uri (the full URI.) + * host (the hostname of the server receiving the request) + * port (the port of the server receiving the request) + * path (the path of the request) + * query (the query string of the request) + * body (the entire request body) + * headers (the headers of the request) + + Backwards compatible matchers: + * url (the `uri` alias) If these options don't work for you, you can also register your own request matcher. This is described in the Advanced section of this README. @@ -163,8 +169,8 @@ with vcr.use_cassette('fixtures/vcr_cassettes/synopsis.yaml') as cass: response = urllib2.urlopen('http://www.zombo.com/').read() # cass should have 1 request inside it assert len(cass) == 1 - # the request url should have been http://www.zombo.com/ - assert cass.requests[0].url == 'http://www.zombo.com/' + # the request uri should have been http://www.zombo.com/ + assert cass.requests[0].uri == 'http://www.zombo.com/' ``` The `Cassette` object exposes the following properties which I consider part of @@ -177,17 +183,23 @@ the API. The fields are as follows: back * `responses_of(request)`: Access the responses that match a given request -The `Request` object has the following properties +The `Request` object has the following properties: - * `url`: The full url of the request, including the protocol. Example: - "http://www.google.com/" - * `path`: The path of the request. For example "/" or "/home.html" + * `uri`: The full uri of the request. Example: "https://google.com/?q=vcrpy" + * `scheme`: The scheme used to make the request (http or https) * `host`: The host of the request, for example "www.google.com" * `port`: The port the request was made on + * `path`: The path of the request. For example "/" or "/home.html" + * `query`: The parsed query string of the request. Sorted list of name, value pairs. * `method` : The method used to make the request, for example "GET" or "POST" - * `protocol`: The protocol used to make the request (http or https) * `body`: The body of the request, usually empty except for POST / PUT / etc + Backwards compatible properties: + + * `url`: The `uri` alias + * `protocol`: The `scheme` alias + + ## Register your own serializer Don't like JSON or YAML? That's OK, VCR.py can serialize to any format you @@ -239,7 +251,7 @@ Finally, register your method with VCR to use your new request matcher. import vcr def jurassic_matcher(r1, r2): - return r1.url == r2.url and 'JURASSIC PARK' in r1.body + return r1.uri == r2.uri and 'JURASSIC PARK' in r1.body my_vcr = vcr.VCR() my_vcr.register_matcher('jurassic', jurassic_matcher) From 750e141b9db7ca07c74a2807e5b1c29b8c6b18f3 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 14 Apr 2014 23:53:38 +0200 Subject: [PATCH 21/58] Added integration tests for matchers --- tests/integration/test_matchers.py | 97 ++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/tests/integration/test_matchers.py b/tests/integration/test_matchers.py index 007a844..5ead41f 100644 --- a/tests/integration/test_matchers.py +++ b/tests/integration/test_matchers.py @@ -3,40 +3,97 @@ import pytest from six.moves.urllib.request import urlopen +DEFAULT_URI = 'http://httpbin.org/get?p1=q1&p2=q2' # base uri for testing + + @pytest.fixture def cassette(tmpdir): - return str(tmpdir.join('test.yml')) + """ + Helper fixture used to prepare the cassete + returns path to the recorded cassette + """ + cassette_path = str(tmpdir.join('test.yml')) + with vcr.use_cassette(cassette_path, record_mode='all'): + urlopen(DEFAULT_URI) + return cassette_path + + +@pytest.mark.parametrize("matcher, matching_uri, not_matching_uri", [ + ('uri', + 'http://httpbin.org/get?p1=q1&p2=q2', + 'http://httpbin.org/get?p2=q2&p1=q1'), + ('scheme', + 'http://google.com/post?a=b', + 'https://httpbin.org/get?p1=q1&p2=q2'), + ('host', + 'https://httpbin.org/post?a=b', + 'http://google.com/get?p1=q1&p2=q2'), + ('port', + 'https://google.com:80/post?a=b', + 'http://httpbin.org:5000/get?p1=q1&p2=q2'), + ('path', + 'https://google.com/get?a=b', + 'http://httpbin.org/post?p1=q1&p2=q2'), + ('query', + 'https://google.com/get?p2=q2&p1=q1', + 'http://httpbin.org/get?p1=q1&a=b') + ]) +def test_matchers(cassette, matcher, matching_uri, not_matching_uri): + # play cassette with default uri + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(DEFAULT_URI) + assert cass.play_count == 1 + + # play cassette with matching on uri + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(matching_uri) + assert cass.play_count == 1 + + # play cassette with not matching on uri, it should fail + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(not_matching_uri) def test_method_matcher(cassette): - # prepare cassete - with vcr.use_cassette(cassette, match_on=['method']) as cass: - urlopen('http://httpbin.org/') - assert len(cass) == 1 - # play cassette with matching on method with vcr.use_cassette(cassette, match_on=['method']) as cass: - urlopen('http://httpbin.org/get') + urlopen('https://google.com/get?a=b') assert cass.play_count == 1 # should fail if method does not match with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): with vcr.use_cassette(cassette, match_on=['method']) as cass: - urlopen('http://httpbin.org/post', data=b'') # is a POST request + # is a POST request + urlopen(DEFAULT_URI, data=b'') -def test_uri_matcher(cassette): - # prepare cassete - with vcr.use_cassette(cassette, match_on=['uri']) as cass: - urlopen('http://httpbin.org/get?p1=q1&p2=q2') - assert len(cass) == 1 - - # play cassette with matching on uri - with vcr.use_cassette(cassette, match_on=['uri']) as cass: - urlopen('http://httpbin.org/get?p1=q1&p2=q2') +@pytest.mark.parametrize("uri", [ + DEFAULT_URI, + 'http://httpbin.org/get?p2=q2&p1=q1', + 'http://httpbin.org/get?p2=q2&p1=q1', +]) +def test_default_matcher_matches(cassette, uri): + with vcr.use_cassette(cassette) as cass: + urlopen(uri) assert cass.play_count == 1 - # should fail if uri does not match + +@pytest.mark.parametrize("uri", [ + 'https://httpbin.org/get?p1=q1&p2=q2', + 'http://google.com/get?p1=q1&p2=q2', + 'http://httpbin.org:5000/get?p1=q1&p2=q2', + 'http://httpbin.org/post?p1=q1&p2=q2', + 'http://httpbin.org/get?p1=q1&a=b' +]) +def test_default_matcher_does_not_match(cassette, uri): with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): - with vcr.use_cassette(cassette, match_on=['uri']) as cass: - urlopen('http://httpbin.org/get?p2=q2&p1=q1') + with vcr.use_cassette(cassette): + urlopen(uri) + + +def test_default_matcher_does_not_match_on_method(cassette): + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette): + # is a POST request + urlopen(DEFAULT_URI, data=b'') From 5354ef781cebc48d3c3f637608f628043f04d056 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Mon, 21 Apr 2014 21:46:14 +0200 Subject: [PATCH 22/58] Formatted setup.py to make flake8 happy --- setup.py | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index d9c195a..6fef312 100644 --- a/setup.py +++ b/setup.py @@ -13,42 +13,46 @@ class PyTest(TestCommand): self.test_suite = True def run_tests(self): - #import here, cause outside the eggs aren't loaded + # import here, cause outside the eggs aren't loaded import pytest errno = pytest.main(self.test_args) sys.exit(errno) -setup(name='vcrpy', - version='0.7.0', - description="Automatically mock your HTTP interactions to simplify and speed up testing", - author='Kevin McCarthy', - author_email='me@kevinmccarthy.org', - url='https://github.com/kevin1024/vcrpy', - packages = [ +setup( + name='vcrpy', + version='0.7.0', + description=( + "Automatically mock your HTTP interactions to simplify and " + "speed up testing" + ), + author='Kevin McCarthy', + author_email='me@kevinmccarthy.org', + url='https://github.com/kevin1024/vcrpy', + packages=[ 'vcr', 'vcr.stubs', 'vcr.compat', 'vcr.persisters', 'vcr.serializers', - ], - package_dir={ + ], + package_dir={ 'vcr': 'vcr', 'vcr.stubs': 'vcr/stubs', 'vcr.compat': 'vcr/compat', 'vcr.persisters': 'vcr/persisters', - }, - install_requires=['PyYAML','contextdecorator','six'], - license='MIT', - tests_require=['pytest','mock'], - cmdclass={'test': PyTest}, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Topic :: Software Development :: Testing', - 'Topic :: Internet :: WWW/HTTP', - 'License :: OSI Approved :: MIT License', - ], + }, + install_requires=['PyYAML', 'contextdecorator', 'six'], + license='MIT', + tests_require=['pytest', 'mock'], + cmdclass={'test': PyTest}, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Topic :: Software Development :: Testing', + 'Topic :: Internet :: WWW/HTTP', + 'License :: OSI Approved :: MIT License', + ], ) From ee28768a31aaa8cf758b02167bb21ccd184e84a9 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 22 Apr 2014 01:00:02 +0200 Subject: [PATCH 23/58] Added migration script for old cassettes --- tests/fixtures/migration/new_cassette.json | 31 ++++++ tests/fixtures/migration/new_cassette.yaml | 15 +++ tests/fixtures/migration/not_cassette.txt | 1 + tests/fixtures/migration/old_cassette.json | 34 +++++++ tests/fixtures/migration/old_cassette.yaml | 18 ++++ tests/unit/test_migration.py | 36 +++++++ vcr/migration.py | 104 +++++++++++++++++++++ 7 files changed, 239 insertions(+) create mode 100644 tests/fixtures/migration/new_cassette.json create mode 100644 tests/fixtures/migration/new_cassette.yaml create mode 100644 tests/fixtures/migration/not_cassette.txt create mode 100644 tests/fixtures/migration/old_cassette.json create mode 100644 tests/fixtures/migration/old_cassette.yaml create mode 100644 tests/unit/test_migration.py create mode 100644 vcr/migration.py diff --git a/tests/fixtures/migration/new_cassette.json b/tests/fixtures/migration/new_cassette.json new file mode 100644 index 0000000..b0e5dab --- /dev/null +++ b/tests/fixtures/migration/new_cassette.json @@ -0,0 +1,31 @@ +[ + { + "request": { + "body": null, + "method": "GET", + "headers": { + "Accept-Encoding": "gzip, deflate, compress", + "Accept": "*/*", + "User-Agent": "python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0" + }, + "uri" : "http://httpbin.org:80/ip" + }, + "response": { + "status": { + "message": "OK", + "code": 200 + }, + "headers": [ + "Access-Control-Allow-Origin: *\r\n", + "Content-Type: application/json\r\n", + "Date: Mon, 21 Apr 2014 23:13:40 GMT\r\n", + "Server: gunicorn/0.17.4\r\n", + "Content-Length: 32\r\n", + "Connection: keep-alive\r\n" + ], + "body": { + "string": "{\n \"origin\": \"217.122.164.194\"\n}" + } + } + } +] diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml new file mode 100644 index 0000000..bf67623 --- /dev/null +++ b/tests/fixtures/migration/new_cassette.yaml @@ -0,0 +1,15 @@ +- request: !!python/object:vcr.request.Request + body: null + headers: !!python/object/apply:__builtin__.frozenset + - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] + - !!python/tuple [User-Agent, python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0] + - !!python/tuple [Accept, '*/*'] + method: GET + uri: http://httpbin.org:80/ip + response: + body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} + headers: [!!python/unicode "Access-Control-Allow-Origin: *\r\n", !!python/unicode "Content-Type: + application/json\r\n", !!python/unicode "Date: Mon, 21 Apr 2014 23:06:09 GMT\r\n", + !!python/unicode "Server: gunicorn/0.17.4\r\n", !!python/unicode "Content-Length: + 32\r\n", !!python/unicode "Connection: keep-alive\r\n"] + status: {code: 200, message: OK} diff --git a/tests/fixtures/migration/not_cassette.txt b/tests/fixtures/migration/not_cassette.txt new file mode 100644 index 0000000..e1d9fc4 --- /dev/null +++ b/tests/fixtures/migration/not_cassette.txt @@ -0,0 +1 @@ +This is not a cassette diff --git a/tests/fixtures/migration/old_cassette.json b/tests/fixtures/migration/old_cassette.json new file mode 100644 index 0000000..22fd9cd --- /dev/null +++ b/tests/fixtures/migration/old_cassette.json @@ -0,0 +1,34 @@ +[ + { + "request": { + "body": null, + "protocol": "http", + "method": "GET", + "headers": { + "Accept-Encoding": "gzip, deflate, compress", + "Accept": "*/*", + "User-Agent": "python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0" + }, + "host": "httpbin.org", + "path": "/ip", + "port": 80 + }, + "response": { + "status": { + "message": "OK", + "code": 200 + }, + "headers": [ + "Access-Control-Allow-Origin: *\r\n", + "Content-Type: application/json\r\n", + "Date: Mon, 21 Apr 2014 23:13:40 GMT\r\n", + "Server: gunicorn/0.17.4\r\n", + "Content-Length: 32\r\n", + "Connection: keep-alive\r\n" + ], + "body": { + "string": "{\n \"origin\": \"217.122.164.194\"\n}" + } + } + } +] diff --git a/tests/fixtures/migration/old_cassette.yaml b/tests/fixtures/migration/old_cassette.yaml new file mode 100644 index 0000000..970823d --- /dev/null +++ b/tests/fixtures/migration/old_cassette.yaml @@ -0,0 +1,18 @@ +- request: !!python/object:vcr.request.Request + body: null + headers: !!python/object/apply:__builtin__.frozenset + - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] + - !!python/tuple [User-Agent, python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0] + - !!python/tuple [Accept, '*/*'] + host: httpbin.org + method: GET + path: /ip + port: 80 + protocol: http + response: + body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} + headers: [!!python/unicode "Access-Control-Allow-Origin: *\r\n", !!python/unicode "Content-Type: + application/json\r\n", !!python/unicode "Date: Mon, 21 Apr 2014 23:06:09 GMT\r\n", + !!python/unicode "Server: gunicorn/0.17.4\r\n", !!python/unicode "Content-Length: + 32\r\n", !!python/unicode "Connection: keep-alive\r\n"] + status: {code: 200, message: OK} diff --git a/tests/unit/test_migration.py b/tests/unit/test_migration.py new file mode 100644 index 0000000..b83d6b1 --- /dev/null +++ b/tests/unit/test_migration.py @@ -0,0 +1,36 @@ +import filecmp +import json +import shutil + +import vcr.migration + + +def test_try_migrate_with_json(tmpdir): + cassette = tmpdir.join('cassette').strpath + shutil.copy('tests/fixtures/migration/old_cassette.json', cassette) + assert vcr.migration.try_migrate(cassette) + with open('tests/fixtures/migration/new_cassette.json', 'r') as f: + expected_json = json.load(f) + with open(cassette, 'r') as f: + actual_json = json.load(f) + assert actual_json == expected_json + + +def test_try_migrate_with_yaml(tmpdir): + cassette = tmpdir.join('cassette').strpath + shutil.copy('tests/fixtures/migration/old_cassette.yaml', cassette) + assert vcr.migration.try_migrate(cassette) + assert filecmp.cmp(cassette, 'tests/fixtures/migration/new_cassette.yaml') + + +def test_try_migrate_with_invalid_or_new_cassettes(tmpdir): + cassette = tmpdir.join('cassette').strpath + files = [ + 'tests/fixtures/migration/not_cassette.txt', + 'tests/fixtures/migration/new_cassette.yaml', + 'tests/fixtures/migration/new_cassette.json', + ] + for file_path in files: + shutil.copy(file_path, cassette) + assert not vcr.migration.try_migrate(cassette) + assert filecmp.cmp(cassette, file_path) # shold not change file diff --git a/vcr/migration.py b/vcr/migration.py new file mode 100644 index 0000000..90a2da9 --- /dev/null +++ b/vcr/migration.py @@ -0,0 +1,104 @@ +""" +Migration script for old 'yaml' and 'json' cassettes + +.. warning:: Backup your cassettes files before migration. + +It merges and deletes the request obsolete keys (protocol, host, port, path) +into new 'uri' key. +Usage:: + + python -m vcr.migration PATH + +The PATH can be path to the directory with cassettes or cassette itself +""" + +from contextlib import closing +import json +import os +import re +import shutil +import sys +import tempfile + + +PARTS = [ + 'protocol', + 'host', + 'port', + 'path', +] + + +def build_uri(**parts): + return "{protocol}://{host}:{port}{path}".format(**parts) + + +def migrate_json(in_fp, out_fp): + data = json.load(in_fp) + for item in data: + req = item['request'] + uri = {k: req.pop(k) for k in PARTS} + req['uri'] = build_uri(**uri) + json.dump(data, out_fp, indent=4) + + +def migrate_yml(in_fp, out_fp): + migrated = False + uri = dict.fromkeys(PARTS, None) + for line in in_fp: + for part in uri: + match = re.match('\s+{}:\s(.*)'.format(part), line) + if match: + uri[part] = match.group(1) + break + else: + out_fp.write(line) + + if None not in uri.values(): # if all uri parts are collected + out_fp.write(" uri: {}\n".format(build_uri(**uri))) + uri = dict.fromkeys(PARTS, None) # reset dict + migrated = True + if not migrated: + raise RuntimeError("migration failed") + + +def migrate(file_path, migration_fn): + # because we assume that original files can be reverted + # we will try to copy the content. (os.rename not needed) + with closing(tempfile.TemporaryFile()) as out_fp: + with open(file_path, 'r') as in_fp: + migration_fn(in_fp, out_fp) + with open(file_path, 'w') as in_fp: + out_fp.seek(0) + shutil.copyfileobj(out_fp, in_fp) + + +def try_migrate(path): + try: # try to migrate as json + migrate(path, migrate_json) + except: # probably the file is not a json + try: # let's try to migrate as yaml + migrate(path, migrate_yml) + except: # oops probably the file is not a cassette + return False + return True + + +def main(): + if len(sys.argv) != 2: + raise SystemExit("Please provide path to cassettes directory or file. " + "Usage: python -m vcr.migration PATH") + + path = sys.argv[1] + if not os.path.isabs(path): + path = os.path.abspath(path) + for root, dirs, files in os.walk(path): + for file_name in files: + file_path = os.path.join(root, file_name) + migrated = try_migrate(file_path) + status = 'OK' if migrated else 'FAIL' + sys.stderr.write("[{}] {}\n".format(status, file_path)) + sys.stderr.write("Done.\n") + +if __name__ == '__main__': + main() From f0972628ef14e46c14a610331320c5454e30e2ef Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 22 Apr 2014 03:09:13 +0200 Subject: [PATCH 24/58] Fixed migration for one file --- vcr/migration.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index 90a2da9..d3e909a 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -92,9 +92,12 @@ def main(): path = sys.argv[1] if not os.path.isabs(path): path = os.path.abspath(path) - for root, dirs, files in os.walk(path): - for file_name in files: - file_path = os.path.join(root, file_name) + files = [path] + if os.path.isdir(path): + files = (os.path.join(root, name) + for (root, dirs, files) in os.walk(path) + for name in files) + for file_path in files: migrated = try_migrate(file_path) status = 'OK' if migrated else 'FAIL' sys.stderr.write("[{}] {}\n".format(status, file_path)) From 424c658da44c8bf926c3c0f69bb7cea91402e388 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 22 Apr 2014 21:32:34 +0200 Subject: [PATCH 25/58] Fixed open tmp file in python3 --- vcr/migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/migration.py b/vcr/migration.py index d3e909a..da1f779 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -65,7 +65,7 @@ def migrate_yml(in_fp, out_fp): def migrate(file_path, migration_fn): # because we assume that original files can be reverted # we will try to copy the content. (os.rename not needed) - with closing(tempfile.TemporaryFile()) as out_fp: + with closing(tempfile.TemporaryFile(mode='w+')) as out_fp: with open(file_path, 'r') as in_fp: migration_fn(in_fp, out_fp) with open(file_path, 'w') as in_fp: From 9d8426e668ac0b185fa6d774fc58b85ca7dbf819 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 22 Apr 2014 23:42:12 +0200 Subject: [PATCH 26/58] Make migration python 2.6 compatible --- vcr/migration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index da1f779..d4fa52c 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -37,7 +37,7 @@ def migrate_json(in_fp, out_fp): data = json.load(in_fp) for item in data: req = item['request'] - uri = {k: req.pop(k) for k in PARTS} + uri = dict((k, req.pop(k)) for k in PARTS) req['uri'] = build_uri(**uri) json.dump(data, out_fp, indent=4) @@ -47,7 +47,7 @@ def migrate_yml(in_fp, out_fp): uri = dict.fromkeys(PARTS, None) for line in in_fp: for part in uri: - match = re.match('\s+{}:\s(.*)'.format(part), line) + match = re.match('\s+{0}:\s(.*)'.format(part), line) if match: uri[part] = match.group(1) break @@ -55,7 +55,7 @@ def migrate_yml(in_fp, out_fp): out_fp.write(line) if None not in uri.values(): # if all uri parts are collected - out_fp.write(" uri: {}\n".format(build_uri(**uri))) + out_fp.write(" uri: {0}\n".format(build_uri(**uri))) uri = dict.fromkeys(PARTS, None) # reset dict migrated = True if not migrated: @@ -100,7 +100,7 @@ def main(): for file_path in files: migrated = try_migrate(file_path) status = 'OK' if migrated else 'FAIL' - sys.stderr.write("[{}] {}\n".format(status, file_path)) + sys.stderr.write("[{0}] {1}\n".format(status, file_path)) sys.stderr.write("Done.\n") if __name__ == '__main__': From 710ec6f432e5c9ac49f28b96e772654516ec24f2 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 23 Apr 2014 00:34:20 +0200 Subject: [PATCH 27/58] Added "New Cassette Format" section to readme --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index e7106ae..d9df355 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,25 @@ If you set the loglevel to DEBUG, you will also get information about which matchers didn't match. This can help you with debugging custom matchers. +## Upgrade + +### New Cassette Format +The cassette format has changed in _VCR.py 1.x_, the _VCR.py 0.x_ cassettes cannot be +used with _VCR.py 1.x_. +The easiest way to upgrade is to simply delete your cassettes and re-record all of them +VCR.py also provides migration script that attempts to upgrade your 0.x cassettes to the +new 1.x format. To use it, run the following command: + +``` +python -m vcr.migration PATH +``` + +The PATH can be path to the directory with cassettes or cassette itself. + +*Note*: Backup your cassettes files before migration. +The migration runs successfully and upgrades the file content only for the old formatted +cassettes. + ## Changelog * 1.0.0 (in development) - Add support for filtering sensitive data from requests, bump supported Python3 version to 3.4, fix some bugs with Boto From a3eac1f0ec0d4084f09e0874c14becea75f5fb5c Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 23 Apr 2014 23:14:59 +0200 Subject: [PATCH 28/58] Added tests for persist module --- tests/unit/test_persist.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/unit/test_persist.py diff --git a/tests/unit/test_persist.py b/tests/unit/test_persist.py new file mode 100644 index 0000000..009d4cb --- /dev/null +++ b/tests/unit/test_persist.py @@ -0,0 +1,14 @@ +import pytest + +import vcr.persist +from vcr.serializers import jsonserializer, yamlserializer + + +@pytest.mark.parametrize("cassette_path, serializer", [ + ('tests/fixtures/migration/old_cassette.json', jsonserializer), + ('tests/fixtures/migration/old_cassette.yaml', yamlserializer), +]) +def test_load_cassette_with_old_cassettes(cassette_path, serializer): + with pytest.raises(ValueError) as excinfo: + vcr.persist.load_cassette(cassette_path, serializer) + assert "run the migration script" in excinfo.exconly() From 9c9612f93e930b6f06eeadbbee52ac193038cf09 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 01:10:47 +0200 Subject: [PATCH 29/58] Fixed crazy/stupid implementation mistakes --- vcr/migration.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index d4fa52c..b377c58 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -12,7 +12,6 @@ Usage:: The PATH can be path to the directory with cassettes or cassette itself """ -from contextlib import closing import json import os import re @@ -65,7 +64,7 @@ def migrate_yml(in_fp, out_fp): def migrate(file_path, migration_fn): # because we assume that original files can be reverted # we will try to copy the content. (os.rename not needed) - with closing(tempfile.TemporaryFile(mode='w+')) as out_fp: + with tempfile.TemporaryFile(mode='w+') as out_fp: with open(file_path, 'r') as in_fp: migration_fn(in_fp, out_fp) with open(file_path, 'w') as in_fp: @@ -76,10 +75,10 @@ def migrate(file_path, migration_fn): def try_migrate(path): try: # try to migrate as json migrate(path, migrate_json) - except: # probably the file is not a json + except Exception: # probably the file is not a json try: # let's try to migrate as yaml migrate(path, migrate_yml) - except: # oops probably the file is not a cassette + except Exception: # oops probably the file is not a cassette return False return True From 0408bdaadbb658e74ac9f3cd475784cdde4f643b Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 02:03:56 +0200 Subject: [PATCH 30/58] Added checkfor old cassette on load cassete to persist module --- vcr/persist.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/vcr/persist.py b/vcr/persist.py index 1f270d8..8c00b32 100644 --- a/vcr/persist.py +++ b/vcr/persist.py @@ -1,9 +1,31 @@ +import tempfile + from .persisters.filesystem import FilesystemPersister +from . import migration + + +def _check_for_old_cassette(cassette_content): + # crate tmp with cassete content and try to migrate it + with tempfile.NamedTemporaryFile(mode='w+') as f: + f.write(cassette_content) + f.seek(0) + if migration.try_migrate(f.name): + raise ValueError( + "Your cassette files were generated in an older version " + "of VCR. Delete your cassettes or run the migration script." + "See http://git.io/mHhLBg for more details." + ) def load_cassette(cassette_path, serializer): with open(cassette_path) as f: - return serializer.deserialize(f.read()) + cassette_content = f.read() + try: + cassette = serializer.deserialize(cassette_content) + except TypeError: + _check_for_old_cassette(cassette_content) + raise + return cassette def save_cassette(cassette_path, cassette_dict, serializer): From 1e995c3c9bbbb5e14ff00c57a3cf0ed6b0e24be2 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 02:05:14 +0200 Subject: [PATCH 31/58] Replaced yaml dump of Request object with plain dict dump --- tests/fixtures/migration/new_cassette.yaml | 8 +++----- tests/fixtures/wild/domain_redirect.yaml | 14 ++++--------- tests/unit/test_cassettes.py | 3 ++- vcr/request.py | 2 +- vcr/serializers/yamlserializer.py | 23 ++++------------------ 5 files changed, 14 insertions(+), 36 deletions(-) diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml index bf67623..649da68 100644 --- a/tests/fixtures/migration/new_cassette.yaml +++ b/tests/fixtures/migration/new_cassette.yaml @@ -1,9 +1,7 @@ -- request: !!python/object:vcr.request.Request +- request: body: null - headers: !!python/object/apply:__builtin__.frozenset - - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] - - !!python/tuple [User-Agent, python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0] - - !!python/tuple [Accept, '*/*'] + headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: python-requests/2.2.1 + CPython/2.6.1 Darwin/10.8.0} method: GET uri: http://httpbin.org:80/ip response: diff --git a/tests/fixtures/wild/domain_redirect.yaml b/tests/fixtures/wild/domain_redirect.yaml index 465f071..7b5173a 100644 --- a/tests/fixtures/wild/domain_redirect.yaml +++ b/tests/fixtures/wild/domain_redirect.yaml @@ -1,9 +1,6 @@ -- request: !!python/object:vcr.request.Request +- request: body: null - headers: !!python/object/apply:__builtin__.frozenset - - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] - - !!python/tuple [User-Agent, vcrpy-test] - - !!python/tuple [Accept, '*/*'] + headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: vcrpy-test} method: GET uri: http://seomoz.org:80/ response: @@ -11,12 +8,9 @@ headers: ["Location: http://moz.com/\r\n", "Server: BigIP\r\n", "Connection: Keep-Alive\r\n", "Content-Length: 0\r\n"] status: {code: 301, message: Moved Permanently} -- request: !!python/object:vcr.request.Request +- request: body: null - headers: !!python/object/apply:__builtin__.frozenset - - - !!python/tuple [Accept-Encoding, 'gzip, deflate, compress'] - - !!python/tuple [User-Agent, vcrpy-test] - - !!python/tuple [Accept, '*/*'] + headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: vcrpy-test} method: GET uri: http://moz.com:80/ response: diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index d351ab0..1c5b84c 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -8,7 +8,8 @@ from vcr.errors import UnhandledHTTPRequestError def test_cassette_load(tmpdir): a_file = tmpdir.join('test_cassette.yml') a_file.write(yaml.dump([ - {'request': 'foo', 'response': 'bar'} + {'request': {'body': '', 'uri': 'foo', 'method': 'GET', 'headers': {}}, + 'response': 'bar'} ])) a_cassette = Cassette.load(str(a_file)) assert len(a_cassette) == 1 diff --git a/vcr/request.py b/vcr/request.py index d9dac9d..a840572 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -71,7 +71,7 @@ class Request(object): 'method': self.method, 'uri': self.uri, 'body': self.body, - 'headers': self.headers, + 'headers': dict(self.headers), } @classmethod diff --git a/vcr/serializers/yamlserializer.py b/vcr/serializers/yamlserializer.py index 76128b2..df42d9c 100644 --- a/vcr/serializers/yamlserializer.py +++ b/vcr/serializers/yamlserializer.py @@ -1,5 +1,6 @@ -import sys import yaml + +from vcr.request import Request from . import compat # Use the libYAML versions if possible @@ -21,25 +22,9 @@ Deserializing: string (yaml converts from utf-8) -> bytestring """ -def _restore_frozenset(): - """ - Restore __builtin__.frozenset for cassettes serialized in python2 but - deserialized in python3 and builtins.frozenset for cassettes serialized - in python3 and deserialized in python2 - """ - - if '__builtin__' not in sys.modules: - import builtins - sys.modules['__builtin__'] = builtins - - if 'builtins' not in sys.modules: - sys.modules['builtins'] = sys.modules['__builtin__'] - - def deserialize(cassette_string): - _restore_frozenset() data = yaml.load(cassette_string, Loader=Loader) - requests = [r['request'] for r in data] + requests = [Request._from_dict(r['request']) for r in data] responses = [r['response'] for r in data] responses = [compat.convert_to_bytes(r['response']) for r in data] return requests, responses @@ -47,7 +32,7 @@ def deserialize(cassette_string): def serialize(cassette_dict): data = ([{ - 'request': request, + 'request': request._to_dict(), 'response': response, } for request, response in zip( cassette_dict['requests'], From 25c0141e2754f8b86ed56580bc72eee4203a03ed Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 02:18:44 +0200 Subject: [PATCH 32/58] Updated migration script to use new yaml serialization --- vcr/migration.py | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index b377c58..947e4f9 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -14,10 +14,19 @@ The PATH can be path to the directory with cassettes or cassette itself import json import os -import re import shutil import sys import tempfile +import yaml + +from .serializers import compat, yamlserializer +from . import request + +# Use the libYAML versions if possible +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader PARTS = [ @@ -41,24 +50,48 @@ def migrate_json(in_fp, out_fp): json.dump(data, out_fp, indent=4) -def migrate_yml(in_fp, out_fp): - migrated = False - uri = dict.fromkeys(PARTS, None) - for line in in_fp: - for part in uri: - match = re.match('\s+{0}:\s(.*)'.format(part), line) - if match: - uri[part] = match.group(1) - break - else: - out_fp.write(line) +def _restore_frozenset(): + """ + Restore __builtin__.frozenset for cassettes serialized in python2 but + deserialized in python3 and builtins.frozenset for cassettes serialized + in python3 and deserialized in python2 + """ - if None not in uri.values(): # if all uri parts are collected - out_fp.write(" uri: {0}\n".format(build_uri(**uri))) - uri = dict.fromkeys(PARTS, None) # reset dict - migrated = True - if not migrated: - raise RuntimeError("migration failed") + if '__builtin__' not in sys.modules: + import builtins + sys.modules['__builtin__'] = builtins + + if 'builtins' not in sys.modules: + sys.modules['builtins'] = sys.modules['__builtin__'] + + +def _old_deserialize(cassette_string): + _restore_frozenset() + data = yaml.load(cassette_string, Loader=Loader) + requests = [r['request'] for r in data] + responses = [r['response'] for r in data] + responses = [compat.convert_to_bytes(r['response']) for r in data] + return requests, responses + + +def migrate_yml(in_fp, out_fp): + (requests, responses) = _old_deserialize(in_fp.read()) + for req in requests: + if not isinstance(req, request.Request): + raise Exception("already migrated") + else: + req.uri = "{0}://{1}:{2}{3}".format( + req.__dict__['protocol'], + req.__dict__['host'], + req.__dict__['port'], + req.__dict__['path'], + ) + + data = yamlserializer.serialize({ + "requests": requests, + "responses": responses, + }) + out_fp.write(data) def migrate(file_path, migration_fn): From 434d6325ea0b01a26a1fbbe8cc6766377cf90acc Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 02:28:30 +0200 Subject: [PATCH 33/58] Udated migration test for yaml. replaced strict content comparision What we care about it is actually data after loading not the strict format of yaml file --- tests/unit/test_migration.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_migration.py b/tests/unit/test_migration.py index b83d6b1..5f2472c 100644 --- a/tests/unit/test_migration.py +++ b/tests/unit/test_migration.py @@ -1,6 +1,7 @@ import filecmp import json import shutil +import yaml import vcr.migration @@ -20,7 +21,11 @@ def test_try_migrate_with_yaml(tmpdir): cassette = tmpdir.join('cassette').strpath shutil.copy('tests/fixtures/migration/old_cassette.yaml', cassette) assert vcr.migration.try_migrate(cassette) - assert filecmp.cmp(cassette, 'tests/fixtures/migration/new_cassette.yaml') + with open('tests/fixtures/migration/new_cassette.yaml', 'r') as f: + expected_yaml = yaml.load(f) + with open(cassette, 'r') as f: + actual_yaml = yaml.load(f) + assert actual_yaml == expected_yaml def test_try_migrate_with_invalid_or_new_cassettes(tmpdir): From eedafb19ee29165f2eea68935d3259915331e423 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Thu, 24 Apr 2014 02:41:35 +0200 Subject: [PATCH 34/58] Added more test for persist --- tests/unit/test_persist.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_persist.py b/tests/unit/test_persist.py index 009d4cb..e9b30be 100644 --- a/tests/unit/test_persist.py +++ b/tests/unit/test_persist.py @@ -12,3 +12,13 @@ def test_load_cassette_with_old_cassettes(cassette_path, serializer): with pytest.raises(ValueError) as excinfo: vcr.persist.load_cassette(cassette_path, serializer) assert "run the migration script" in excinfo.exconly() + + +@pytest.mark.parametrize("cassette_path, serializer", [ + ('tests/fixtures/migration/not_cassette.txt', jsonserializer), + ('tests/fixtures/migration/not_cassette.txt', yamlserializer), +]) +def test_load_cassette_with_invalid_cassettes(cassette_path, serializer): + with pytest.raises(Exception) as excinfo: + vcr.persist.load_cassette(cassette_path, serializer) + assert "run the migration script" not in excinfo.exconly() From b6195bf41efa1097eeacb79c6a7488275c15b23e Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sat, 26 Apr 2014 22:29:34 +0200 Subject: [PATCH 35/58] Updated migration fixtures in correspondence with new fixture format --- tests/fixtures/migration/new_cassette.json | 16 ++++++++-------- tests/fixtures/migration/new_cassette.yaml | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/fixtures/migration/new_cassette.json b/tests/fixtures/migration/new_cassette.json index b0e5dab..166c4fc 100644 --- a/tests/fixtures/migration/new_cassette.json +++ b/tests/fixtures/migration/new_cassette.json @@ -1,15 +1,15 @@ [ { "request": { - "body": null, - "method": "GET", + "body": null, "headers": { - "Accept-Encoding": "gzip, deflate, compress", - "Accept": "*/*", - "User-Agent": "python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0" - }, - "uri" : "http://httpbin.org:80/ip" - }, + "Accept": ["*/*"], + "Accept-Encoding": ["gzip, deflate, compress"], + "User-Agent": ["python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0"] + }, + "method": "GET", + "uri": "http://httpbin.org:80/ip" + }, "response": { "status": { "message": "OK", diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml index 649da68..2adcbe4 100644 --- a/tests/fixtures/migration/new_cassette.yaml +++ b/tests/fixtures/migration/new_cassette.yaml @@ -1,7 +1,9 @@ - request: body: null - headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: python-requests/2.2.1 - CPython/2.6.1 Darwin/10.8.0} + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate, compress'] + User-Agent: ['python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0'] method: GET uri: http://httpbin.org:80/ip response: From 34ce0a35ecccb09a3f8578959b9560fb2405a5b0 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 29 Apr 2014 22:49:50 +0200 Subject: [PATCH 36/58] Updated wild fixtures in correspondence with new fixture format --- tests/fixtures/wild/domain_redirect.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/wild/domain_redirect.yaml b/tests/fixtures/wild/domain_redirect.yaml index 7b5173a..027bbee 100644 --- a/tests/fixtures/wild/domain_redirect.yaml +++ b/tests/fixtures/wild/domain_redirect.yaml @@ -1,6 +1,9 @@ - request: body: null - headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: vcrpy-test} + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate, compress'] + User-Agent: ['vcrpy-test'] method: GET uri: http://seomoz.org:80/ response: @@ -10,7 +13,10 @@ status: {code: 301, message: Moved Permanently} - request: body: null - headers: {Accept: '*/*', Accept-Encoding: 'gzip, deflate, compress', User-Agent: vcrpy-test} + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate, compress'] + User-Agent: ['vcrpy-test'] method: GET uri: http://moz.com:80/ response: From 7e677f516d023a21e2f2bf6db34a9ca536370591 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 29 Apr 2014 23:25:14 +0200 Subject: [PATCH 37/58] Deleted unnecessary __hash__ method of Request --- vcr/request.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/vcr/request.py b/vcr/request.py index a840572..5415743 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -46,20 +46,6 @@ class Request(object): def protocol(self): return self.scheme - def __key(self): - return ( - self.method, - self.uri, - self.body, - self.headers - ) - - def __hash__(self): - return hash(self.__key()) - - def __eq__(self, other): - return hash(self) == hash(other) - def __str__(self): return "".format(self.method, self.uri) From e4d1db061722528e0317ff1578bef64cf8ff6942 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Tue, 29 Apr 2014 23:59:25 +0200 Subject: [PATCH 38/58] Removed frozenset --- vcr/request.py | 7 ++----- vcr/serializers/jsonserializer.py | 10 ++-------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/vcr/request.py b/vcr/request.py index 5415743..10b9fb5 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -7,13 +7,10 @@ class Request(object): self.method = method self.uri = uri self.body = body - # make headers a frozenset so it will be hashable - self.headers = frozenset(headers.items()) + self.headers = headers def add_header(self, key, value): - tmp = dict(self.headers) - tmp[key] = value - self.headers = frozenset(tmp.iteritems()) + self.headers[key] = value @property def scheme(self): diff --git a/vcr/serializers/jsonserializer.py b/vcr/serializers/jsonserializer.py index eeaa91b..2dcfa09 100644 --- a/vcr/serializers/jsonserializer.py +++ b/vcr/serializers/jsonserializer.py @@ -6,12 +6,6 @@ except ImportError: import json -def _json_default(obj): - if isinstance(obj, frozenset): - return dict(obj) - return obj - - def deserialize(cassette_string): data = json.loads(cassette_string) requests = [Request._from_dict(r['request']) for r in data] @@ -28,8 +22,8 @@ def serialize(cassette_dict): cassette_dict['responses'] )]) try: - return json.dumps(data, indent=4, default=_json_default) - except UnicodeDecodeError as e: + return json.dumps(data, indent=4) + except UnicodeDecodeError: raise UnicodeDecodeError( "Error serializing cassette to JSON. ", "Does this HTTP interaction contain binary data? ", From eab10578d55255bff54d543bbdeff1c3bb9c871b Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 01:18:12 +0200 Subject: [PATCH 39/58] Make Request headers to be a dict of lists --- tests/unit/test_request.py | 15 +++++++++++++++ vcr/request.py | 12 +++++++++--- vcr/stubs/__init__.py | 2 +- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index adecee6..fdc2e89 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -4,3 +4,18 @@ from vcr.request import Request def test_str(): req = Request('GET', 'http://www.google.com:80/', '', {}) str(req) == '' + + +def test_headers(): + headers = {'X-Header1': ['h1'], 'X-Header2': 'h2'} + req = Request('GET', 'http://go.com:80/', '', headers) + assert req.headers == {'X-Header1': ['h1'], 'X-Header2': ['h2']} + + req.add_header('X-Header1', 'h11') + assert req.headers == {'X-Header1': ['h1', 'h11'], 'X-Header2': ['h2']} + + +def test_flat_headers_dict(): + headers = {'X-Header1': ['h1', 'h11'], 'X-Header2': ['h2']} + req = Request('GET', 'http://go.com:80/', '', headers) + assert req.flat_headers_dict() == {'X-Header1': 'h1', 'X-Header2': 'h2'} diff --git a/vcr/request.py b/vcr/request.py index 10b9fb5..6d48a56 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -7,10 +7,16 @@ class Request(object): self.method = method self.uri = uri self.body = body - self.headers = headers + self.headers = {} + for key in headers: + self.add_header(key, headers[key]) def add_header(self, key, value): - self.headers[key] = value + value = list(value) if isinstance(value, (tuple, list)) else [value] + self.headers.setdefault(key, []).extend(value) + + def flat_headers_dict(self): + return {key: self.headers[key][0] for key in self.headers} @property def scheme(self): @@ -54,7 +60,7 @@ class Request(object): 'method': self.method, 'uri': self.uri, 'body': self.body, - 'headers': dict(self.headers), + 'headers': self.headers, } @classmethod diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 77391b2..fc14919 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -225,7 +225,7 @@ class VCRConnection: method=self._vcr_request.method, url=self._url(self._vcr_request.uri), body=self._vcr_request.body, - headers=dict(self._vcr_request.headers or {}) + headers=self._vcr_request.flat_headers_dict(), ) # get the response From fbb6382c128768f381e1935ca8196558c17d7e3c Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 01:43:47 +0200 Subject: [PATCH 40/58] Added migration for Request headers to be a dict of lists --- vcr/migration.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vcr/migration.py b/vcr/migration.py index 947e4f9..a06e9ad 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -47,6 +47,10 @@ def migrate_json(in_fp, out_fp): req = item['request'] uri = dict((k, req.pop(k)) for k in PARTS) req['uri'] = build_uri(**uri) + # convert headers to dict of lists + headers = req['headers'] + for k in headers: + headers[k] = [headers[k]] json.dump(data, out_fp, indent=4) @@ -87,6 +91,12 @@ def migrate_yml(in_fp, out_fp): req.__dict__['path'], ) + # convert headers to dict of lists + headers = req.headers + req.headers = {} + for key, value in headers: + req.add_header(key, value) + data = yamlserializer.serialize({ "requests": requests, "responses": responses, From 5d1f35973dcfb0afe76ce2144614edfa345e16b6 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 02:14:34 +0200 Subject: [PATCH 41/58] Make code 2.6 compatible --- vcr/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/request.py b/vcr/request.py index 6d48a56..8251769 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -16,7 +16,7 @@ class Request(object): self.headers.setdefault(key, []).extend(value) def flat_headers_dict(self): - return {key: self.headers[key][0] for key in self.headers} + return dict((key, self.headers[key][0]) for key in self.headers) @property def scheme(self): From 7b253ebc6fffa4f881b854dd4e302f39a873f624 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 02:41:21 +0200 Subject: [PATCH 42/58] pep8 --- vcr/cassette.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index f71bf03..4053c25 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -1,10 +1,9 @@ '''The container for recorded requests and responses''' try: - from collections import Counter, OrderedDict + from collections import Counter except ImportError: from .compat.counter import Counter - from .compat.ordereddict import OrderedDict from contextdecorator import ContextDecorator @@ -31,7 +30,7 @@ class Cassette(ContextDecorator): path, serializer=yamlserializer, record_mode='once', - match_on=[uri, method]): + match_on=[uri, method], filter_headers=[], filter_query_parameters=[], before_record=None, @@ -70,10 +69,10 @@ class Cassette(ContextDecorator): def append(self, request, response): '''Add a request, response pair to this cassette''' request = filter_request( - request = request, - filter_headers = self._filter_headers, - filter_query_parameters = self._filter_query_parameters, - before_record = self._before_record + request=request, + filter_headers=self._filter_headers, + filter_query_parameters=self._filter_query_parameters, + before_record=self._before_record ) if not request: return @@ -86,10 +85,10 @@ class Cassette(ContextDecorator): the request. """ request = filter_request( - request = request, - filter_headers = self._filter_headers, - filter_query_parameters = self._filter_query_parameters, - before_record = self._before_record + request=request, + filter_headers=self._filter_headers, + filter_query_parameters=self._filter_query_parameters, + before_record=self._before_record ) if not request: return From fbd5049d389ec513fe96c18afaead76aca0fcf85 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 02:49:55 +0200 Subject: [PATCH 43/58] Updated test to use new Request constructor --- tests/unit/test_json_serializer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_json_serializer.py b/tests/unit/test_json_serializer.py index fe043c2..9ab9093 100644 --- a/tests/unit/test_json_serializer.py +++ b/tests/unit/test_json_serializer.py @@ -4,8 +4,13 @@ from vcr.request import Request def test_serialize_binary(): - request = Request('http','localhost',80,'GET','/',{},{}) - cassette = {'requests': [request], 'responses': [{'body':b'\x8c'}]} + request = Request( + method='GET', + uri='htt://localhost:80/', + body='', + headers={}, + ) + cassette = {'requests': [request], 'responses': [{'body': b'\x8c'}]} with pytest.raises(Exception) as e: serialize(cassette) From a48f621bae3d25e9a1ea3cbc0df833446d43ea64 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 03:02:24 +0200 Subject: [PATCH 44/58] Updated _remove_headers to use latest Headers structure Probably we need API in Request object like 'remove_header' --- vcr/filters.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index 06838d6..0e7d106 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -3,11 +3,10 @@ import copy def _remove_headers(request, headers_to_remove): - out = [] - for k, v in request.headers: - if k.lower() not in [h.lower() for h in headers_to_remove]: - out.append((k, v)) - request.headers = frozenset(out) + headers_to_remove = [h.lower() for h in headers_to_remove] + keys = [k for k in request.headers if k.lower() in headers_to_remove] + for k in keys: + request.headers.pop(k) return request From faa83b9aba37649b8822e6e1cda6f34257ecfa23 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 03:04:54 +0200 Subject: [PATCH 45/58] Fixed name of the variable --- vcr/filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index 0e7d106..627dbd4 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse, parse_qsl, urlunparse, urlencode +from six.moves.urllib.parse import urlparse, parse_qsl, urlencode import copy @@ -11,7 +11,7 @@ def _remove_headers(request, headers_to_remove): def _remove_query_parameters(request, query_parameters_to_remove): - if not hasattr(request, 'path' or not query_parameters_to_remote): + if not hasattr(request, 'path' or not query_parameters_to_remove): return request url = urlparse(request.url) q = parse_qsl(url.query) From 998dde61ecab4694fe156255ec2550727371b77e Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 03:29:09 +0200 Subject: [PATCH 46/58] Updated _remove_query_parameters to use latest Request Api --- vcr/filters.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index 627dbd4..b9c8b0e 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse, parse_qsl, urlencode +from six.moves.urllib.parse import urlparse, urlencode, urlunparse import copy @@ -11,15 +11,14 @@ def _remove_headers(request, headers_to_remove): def _remove_query_parameters(request, query_parameters_to_remove): - if not hasattr(request, 'path' or not query_parameters_to_remove): - return request - url = urlparse(request.url) - q = parse_qsl(url.query) - q = [(k, v) for k, v in q if k not in query_parameters_to_remove] - if q: - request.path = url.path + '?' + urlencode(q) - else: - request.path = url.path + query = request.query + new_query = [(k, v) for (k, v) in query + if k not in query_parameters_to_remove] + if len(new_query) != len(query): + uri_parts = list(urlparse(request.uri)) + uri_parts[4] = urlencode(new_query) + request.uri = urlunparse(uri_parts) + print urlunparse(uri_parts) return request From 65c2797f942a1f9c3a94bb82928f20ffc080427f Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 03:48:29 +0200 Subject: [PATCH 47/58] Updated test for filters. Mock replaced with real Request object --- tests/unit/test_filters.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 5d6ac5f..f274279 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -1,28 +1,37 @@ -import mock from vcr.filters import _remove_headers, _remove_query_parameters from vcr.request import Request def test_remove_headers(): - request = mock.Mock(headers=[('hello','goodbye'),('secret','header')]) - assert _remove_headers(request, ['secret']).headers == frozenset([('hello','goodbye')]) + headers = {'hello': ['goodbye'], 'secret': ['header']} + request = Request('GET', 'http://google.com', '', headers) + _remove_headers(request, ['secret']) + assert request.headers == {'hello': ['goodbye']} def test_remove_headers_empty(): - request = mock.Mock(headers=[('hello','goodbye'),('secret','header')]) - assert _remove_headers(request, []).headers == frozenset([('hello','goodbye'),('secret','header')]) + headers = {'hello': ['goodbye'], 'secret': ['header']} + request = Request('GET', 'http://google.com', '', headers) + _remove_headers(request, []) + assert request.headers == headers def test_remove_query_parameters(): - request = mock.Mock(url='http://g.com/?q=cowboys&w=1') - assert _remove_query_parameters(request, ['w']).path == '/?q=cowboys' + uri = 'http://g.com/?q=cowboys&w=1' + request = Request('GET', uri, '', {}) + _remove_query_parameters(request, ['w']) + assert request.uri == 'http://g.com/?q=cowboys' def test_remove_all_query_parameters(): - request = mock.Mock(url='http://g.com/?q=cowboys&w=1') - assert _remove_query_parameters(request, ['w','q']).path == '/' + uri = 'http://g.com/?q=cowboys&w=1' + request = Request('GET', uri, '', {}) + _remove_query_parameters(request, ['w', 'q']) + assert request.uri == 'http://g.com/' def test_remove_nonexistent_query_parameters(): - request = mock.Mock(url='http://g.com/') - assert _remove_query_parameters(request, ['w','q']).path == '/' + uri = 'http://g.com/' + request = Request('GET', uri, '', {}) + _remove_query_parameters(request, ['w', 'q']) + assert request.uri == 'http://g.com/' From 3d2da2693322efcf27ea53fb3ec89fcefd4366f4 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 04:05:27 +0200 Subject: [PATCH 48/58] Updated test to use new headers structure --- tests/integration/test_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_filter.py b/tests/integration/test_filter.py index d1bad9c..11dea94 100644 --- a/tests/integration/test_filter.py +++ b/tests/integration/test_filter.py @@ -16,7 +16,7 @@ def _request_with_auth(url, username, password): def _find_header(cassette, header): for request in cassette.requests: - for k, v in request.headers: + for k in request.headers: if header.lower() == k.lower(): return True return False @@ -25,7 +25,7 @@ def _find_header(cassette, header): def test_filter_basic_auth(tmpdir): url = 'http://httpbin.org/basic-auth/user/passwd' cass_file = str(tmpdir.join('basic_auth_filter.yaml')) - my_vcr = vcr.VCR(match_on = ['url', 'method', 'headers']) + my_vcr = vcr.VCR(match_on=['uri', 'method', 'headers']) # 2 requests, one with auth failure and one with auth success with my_vcr.use_cassette(cass_file, filter_headers=['authorization']): with pytest.raises(HTTPError): From 62d19e5cc1fceb5b84c5faf80c2f189004f54e48 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 04:06:14 +0200 Subject: [PATCH 49/58] Update _remove_headers to work with headers copy Because of the filter implementation here we nedd to work only with clone of the headers and request. subj to refactor --- vcr/filters.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vcr/filters.py b/vcr/filters.py index b9c8b0e..0b9802a 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -3,10 +3,13 @@ import copy def _remove_headers(request, headers_to_remove): + headers = copy.copy(request.headers) headers_to_remove = [h.lower() for h in headers_to_remove] - keys = [k for k in request.headers if k.lower() in headers_to_remove] - for k in keys: - request.headers.pop(k) + keys = [k for k in headers if k.lower() in headers_to_remove] + if keys: + for k in keys: + headers.pop(k) + request.headers = headers return request From 1d8e2dbb4176dafc4cb67acd0bc257ecdb2e5866 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 04:20:27 +0200 Subject: [PATCH 50/58] Removed accidental print call --- vcr/filters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vcr/filters.py b/vcr/filters.py index 0b9802a..318d651 100644 --- a/vcr/filters.py +++ b/vcr/filters.py @@ -21,7 +21,6 @@ def _remove_query_parameters(request, query_parameters_to_remove): uri_parts = list(urlparse(request.uri)) uri_parts[4] = urlencode(new_query) request.uri = urlunparse(uri_parts) - print urlunparse(uri_parts) return request From 1ff5d08c8be25f7d85dbb9a5f64c14ce1e2bb015 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Wed, 30 Apr 2014 23:32:15 +0200 Subject: [PATCH 51/58] Fixed typo --- tests/unit/test_json_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_json_serializer.py b/tests/unit/test_json_serializer.py index 9ab9093..0787d0b 100644 --- a/tests/unit/test_json_serializer.py +++ b/tests/unit/test_json_serializer.py @@ -6,7 +6,7 @@ from vcr.request import Request def test_serialize_binary(): request = Request( method='GET', - uri='htt://localhost:80/', + uri='http://localhost:80/', body='', headers={}, ) From 61e3bdc40239cfb5e83b9ef068c2dfa97b9fafa5 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sat, 3 May 2014 22:31:30 +0200 Subject: [PATCH 52/58] Added tetst for uri and port of Request --- tests/unit/test_request.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index fdc2e89..ce0186f 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -19,3 +19,19 @@ def test_flat_headers_dict(): headers = {'X-Header1': ['h1', 'h11'], 'X-Header2': ['h2']} req = Request('GET', 'http://go.com:80/', '', headers) assert req.flat_headers_dict() == {'X-Header1': 'h1', 'X-Header2': 'h2'} + + +def test_port(): + req = Request('GET', 'http://go.com/', '', {}) + assert req.port == 80 + + req = Request('GET', 'http://go.com:3000/', '', {}) + assert req.port == 3000 + + +def test_uri(): + req = Request('GET', 'http://go.com/', '', {}) + assert req.uri == 'http://go.com/' + + req = Request('GET', 'http://go.com:80/', '', {}) + assert req.uri == 'http://go.com:80/' From 1190a0e62ece01570667e804b906f93219b5f1e8 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sat, 3 May 2014 22:32:27 +0200 Subject: [PATCH 53/58] Removed default '80' port of uri in tests --- tests/fixtures/migration/new_cassette.json | 2 +- tests/fixtures/migration/new_cassette.yaml | 2 +- tests/fixtures/wild/domain_redirect.yaml | 4 ++-- tests/integration/test_request.py | 4 ++-- tests/unit/test_json_serializer.py | 2 +- tests/unit/test_matchers.py | 20 ++++++++++---------- tests/unit/test_request.py | 8 ++++---- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/fixtures/migration/new_cassette.json b/tests/fixtures/migration/new_cassette.json index 166c4fc..acf1267 100644 --- a/tests/fixtures/migration/new_cassette.json +++ b/tests/fixtures/migration/new_cassette.json @@ -8,7 +8,7 @@ "User-Agent": ["python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0"] }, "method": "GET", - "uri": "http://httpbin.org:80/ip" + "uri": "http://httpbin.org/ip" }, "response": { "status": { diff --git a/tests/fixtures/migration/new_cassette.yaml b/tests/fixtures/migration/new_cassette.yaml index 2adcbe4..58d19ea 100644 --- a/tests/fixtures/migration/new_cassette.yaml +++ b/tests/fixtures/migration/new_cassette.yaml @@ -5,7 +5,7 @@ Accept-Encoding: ['gzip, deflate, compress'] User-Agent: ['python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0'] method: GET - uri: http://httpbin.org:80/ip + uri: http://httpbin.org/ip response: body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} headers: [!!python/unicode "Access-Control-Allow-Origin: *\r\n", !!python/unicode "Content-Type: diff --git a/tests/fixtures/wild/domain_redirect.yaml b/tests/fixtures/wild/domain_redirect.yaml index 027bbee..e7d4224 100644 --- a/tests/fixtures/wild/domain_redirect.yaml +++ b/tests/fixtures/wild/domain_redirect.yaml @@ -5,7 +5,7 @@ Accept-Encoding: ['gzip, deflate, compress'] User-Agent: ['vcrpy-test'] method: GET - uri: http://seomoz.org:80/ + uri: http://seomoz.org/ response: body: {string: ''} headers: ["Location: http://moz.com/\r\n", "Server: BigIP\r\n", "Connection: Keep-Alive\r\n", @@ -18,7 +18,7 @@ Accept-Encoding: ['gzip, deflate, compress'] User-Agent: ['vcrpy-test'] method: GET - uri: http://moz.com:80/ + uri: http://moz.com/ response: body: string: !!binary | diff --git a/tests/integration/test_request.py b/tests/integration/test_request.py index 882783e..829c6eb 100644 --- a/tests/integration/test_request.py +++ b/tests/integration/test_request.py @@ -6,6 +6,6 @@ def test_recorded_request_url_with_redirected_request(tmpdir): with vcr.use_cassette(str(tmpdir.join('test.yml'))) as cass: assert len(cass) == 0 urlopen('http://httpbin.org/redirect/3') - assert cass.requests[0].url == 'http://httpbin.org:80/redirect/3' - assert cass.requests[3].url == 'http://httpbin.org:80/get' + assert cass.requests[0].url == 'http://httpbin.org/redirect/3' + assert cass.requests[3].url == 'http://httpbin.org/get' assert len(cass) == 4 diff --git a/tests/unit/test_json_serializer.py b/tests/unit/test_json_serializer.py index 0787d0b..3bf927a 100644 --- a/tests/unit/test_json_serializer.py +++ b/tests/unit/test_json_serializer.py @@ -6,7 +6,7 @@ from vcr.request import Request def test_serialize_binary(): request = Request( method='GET', - uri='http://localhost:80/', + uri='http://localhost/', body='', headers={}, ) diff --git a/tests/unit/test_matchers.py b/tests/unit/test_matchers.py index 0d53fd7..f942bd2 100644 --- a/tests/unit/test_matchers.py +++ b/tests/unit/test_matchers.py @@ -6,13 +6,13 @@ from vcr import request # the dict contains requests with corresponding to its key difference # with 'base' request. REQUESTS = { - 'base': request.Request('GET', 'http://host.com:80/p?a=b', '', {}), - 'method': request.Request('POST', 'http://host.com:80/p?a=b', '', {}), + 'base': request.Request('GET', 'http://host.com/p?a=b', '', {}), + 'method': request.Request('POST', 'http://host.com/p?a=b', '', {}), 'scheme': request.Request('GET', 'https://host.com:80/p?a=b', '', {}), - 'host': request.Request('GET', 'http://another-host.com:80/p?a=b', '', {}), + 'host': request.Request('GET', 'http://another-host.com/p?a=b', '', {}), 'port': request.Request('GET', 'http://host.com:90/p?a=b', '', {}), - 'path': request.Request('GET', 'http://host.com:80/x?a=b', '', {}), - 'query': request.Request('GET', 'http://host.com:80/p?c=d', '', {}), + 'path': request.Request('GET', 'http://host.com/x?a=b', '', {}), + 'query': request.Request('GET', 'http://host.com/p?c=d', '', {}), } @@ -36,13 +36,13 @@ def test_uri_matcher(): def test_query_matcher(): - req1 = request.Request('GET', 'http://host.com:80/?a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com:80/?c=d&a=b', '', {}) + req1 = request.Request('GET', 'http://host.com/?a=b&c=d', '', {}) + req2 = request.Request('GET', 'http://host.com/?c=d&a=b', '', {}) assert matchers.query(req1, req2) - req1 = request.Request('GET', 'http://host.com:80/?a=b&a=b&c=d', '', {}) - req2 = request.Request('GET', 'http://host.com:80/?a=b&c=d&a=b', '', {}) - req3 = request.Request('GET', 'http://host.com:80/?c=d&a=b&a=b', '', {}) + req1 = request.Request('GET', 'http://host.com/?a=b&a=b&c=d', '', {}) + req2 = request.Request('GET', 'http://host.com/?a=b&c=d&a=b', '', {}) + req3 = request.Request('GET', 'http://host.com/?c=d&a=b&a=b', '', {}) assert matchers.query(req1, req2) assert matchers.query(req1, req3) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index ce0186f..13f5271 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -2,13 +2,13 @@ from vcr.request import Request def test_str(): - req = Request('GET', 'http://www.google.com:80/', '', {}) - str(req) == '' + req = Request('GET', 'http://www.google.com/', '', {}) + str(req) == '' def test_headers(): headers = {'X-Header1': ['h1'], 'X-Header2': 'h2'} - req = Request('GET', 'http://go.com:80/', '', headers) + req = Request('GET', 'http://go.com/', '', headers) assert req.headers == {'X-Header1': ['h1'], 'X-Header2': ['h2']} req.add_header('X-Header1', 'h11') @@ -17,7 +17,7 @@ def test_headers(): def test_flat_headers_dict(): headers = {'X-Header1': ['h1', 'h11'], 'X-Header2': ['h2']} - req = Request('GET', 'http://go.com:80/', '', headers) + req = Request('GET', 'http://go.com/', '', headers) assert req.flat_headers_dict() == {'X-Header1': 'h1', 'X-Header2': 'h2'} From 0b1aeac25ecdfb5bc56474ca8d7657de9d6581ab Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 4 May 2014 00:01:30 +0200 Subject: [PATCH 54/58] Renamed outdated url to uri. --- tests/integration/test_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_request.py b/tests/integration/test_request.py index 829c6eb..0e01674 100644 --- a/tests/integration/test_request.py +++ b/tests/integration/test_request.py @@ -2,10 +2,10 @@ import vcr from six.moves.urllib.request import urlopen -def test_recorded_request_url_with_redirected_request(tmpdir): +def test_recorded_request_uri_with_redirected_request(tmpdir): with vcr.use_cassette(str(tmpdir.join('test.yml'))) as cass: assert len(cass) == 0 urlopen('http://httpbin.org/redirect/3') - assert cass.requests[0].url == 'http://httpbin.org/redirect/3' - assert cass.requests[3].url == 'http://httpbin.org/get' + assert cass.requests[0].uri == 'http://httpbin.org/redirect/3' + assert cass.requests[3].uri == 'http://httpbin.org/get' assert len(cass) == 4 From 5d10a38160a52dcc03f133f69230634c8eea118e Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 4 May 2014 02:05:55 +0200 Subject: [PATCH 55/58] Updated migration to support default ports --- vcr/migration.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/vcr/migration.py b/vcr/migration.py index a06e9ad..edb3b91 100644 --- a/vcr/migration.py +++ b/vcr/migration.py @@ -38,7 +38,11 @@ PARTS = [ def build_uri(**parts): - return "{protocol}://{host}:{port}{path}".format(**parts) + port = parts['port'] + scheme = parts['protocol'] + default_port = {'https': 433, 'http': 80}[scheme] + parts['port'] = ':{0}'.format(port) if port != default_port else '' + return "{protocol}://{host}{port}{path}".format(**parts) def migrate_json(in_fp, out_fp): @@ -84,11 +88,11 @@ def migrate_yml(in_fp, out_fp): if not isinstance(req, request.Request): raise Exception("already migrated") else: - req.uri = "{0}://{1}:{2}{3}".format( - req.__dict__['protocol'], - req.__dict__['host'], - req.__dict__['port'], - req.__dict__['path'], + req.uri = build_uri( + protocol=req.__dict__['protocol'], + host=req.__dict__['host'], + port=req.__dict__['port'], + path=req.__dict__['path'], ) # convert headers to dict of lists From 3322234b25d82f7c82a8fc50cf0474d8ba332aff Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 4 May 2014 02:06:21 +0200 Subject: [PATCH 56/58] Updated Request with stup to support default ports --- vcr/request.py | 6 +++++- vcr/stubs/__init__.py | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/vcr/request.py b/vcr/request.py index 8251769..417c40c 100644 --- a/vcr/request.py +++ b/vcr/request.py @@ -28,7 +28,11 @@ class Request(object): @property def port(self): - return urlparse(self.uri).port + parse_uri = urlparse(self.uri) + port = parse_uri.port + if port is None: + port = {'https': 433, 'http': 80}[parse_uri.scheme] + return port @property def path(self): diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index fc14919..2218786 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -119,21 +119,30 @@ class VCRConnection: # A reference to the cassette that's currently being patched in cassette = None + def _port_postfix(self): + """ + Returns empty string for the default port and ':port' otherwise + """ + port = self.real_connection.port + default_port = {'https': 433, 'http': 80}[self._protocol] + return ':{0}'.format(port) if port != default_port else '' + def _uri(self, url): """Returns request absolute URI""" - return "{0}://{1}:{2}{3}".format( + uri = "{0}://{1}{2}{3}".format( self._protocol, self.real_connection.host, - self.real_connection.port, + self._port_postfix(), url, ) + return uri def _url(self, uri): """Returns request selector url from absolute URI""" - prefix = "{0}://{1}:{2}".format( + prefix = "{0}://{1}{2}".format( self._protocol, self.real_connection.host, - self.real_connection.port, + self._port_postfix(), ) return uri.replace(prefix, '', 1) From ce5d2225a6bcc7b368c3febfa6e0204d5e762dcc Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 4 May 2014 02:10:19 +0200 Subject: [PATCH 57/58] pep8 --- vcr/matchers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vcr/matchers.py b/vcr/matchers.py index c471538..4964b9f 100644 --- a/vcr/matchers.py +++ b/vcr/matchers.py @@ -1,6 +1,7 @@ import logging log = logging.getLogger(__name__) + def method(r1, r2): return r1.method == r2.method @@ -40,7 +41,11 @@ def headers(r1, r2): def _log_matches(matches): differences = [m for m in matches if not m[0]] if differences: - log.debug('Requests differ according to the following matchers: {0}'.format(differences)) + log.debug( + 'Requests differ according to the following matchers: ' + + str(differences) + ) + def requests_match(r1, r2, matchers): matches = [(m(r1, r2), m) for m in matchers] From 78f6ce46b56634fcf230fdc0a8f8ae7759cd0a55 Mon Sep 17 00:00:00 2001 From: Max Shytikov Date: Sun, 4 May 2014 02:20:46 +0200 Subject: [PATCH 58/58] Added test casses and refactored test for Request#port --- tests/unit/test_request.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index 13f5271..ae0685e 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -1,3 +1,5 @@ +import pytest + from vcr.request import Request @@ -21,12 +23,17 @@ def test_flat_headers_dict(): assert req.flat_headers_dict() == {'X-Header1': 'h1', 'X-Header2': 'h2'} -def test_port(): - req = Request('GET', 'http://go.com/', '', {}) - assert req.port == 80 - - req = Request('GET', 'http://go.com:3000/', '', {}) - assert req.port == 3000 +@pytest.mark.parametrize("uri, expected_port", [ + ('http://go.com/', 80), + ('http://go.com:80/', 80), + ('http://go.com:3000/', 3000), + ('https://go.com/', 433), + ('https://go.com:433/', 433), + ('https://go.com:3000/', 3000), + ]) +def test_port(uri, expected_port): + req = Request('GET', uri, '', {}) + assert req.port == expected_port def test_uri():