From 63ec95be069c3605c28730654b19959d8326ea5e Mon Sep 17 00:00:00 2001 From: aisch Date: Mon, 23 Mar 2015 12:12:49 -0700 Subject: [PATCH 1/3] update urllib3 patch/stub to be same as used for requests and add tests --- .travis.yml | 6 ++ tests/integration/test_urllib3.py | 153 ++++++++++++++++++++++++++++++ tox.ini | 4 +- vcr/patch.py | 77 ++++++++------- vcr/stubs/urllib3_stubs.py | 11 ++- 5 files changed, 208 insertions(+), 43 deletions(-) create mode 100644 tests/integration/test_urllib3.py diff --git a/.travis.yml b/.travis.yml index 09eb1ed..5f17fc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ env: - WITH_LIB="requests1.x" - WITH_LIB="httplib2" - WITH_LIB="boto" + - WITH_LIB="urllib31.7" + - WITH_LIB="urllib31.9" + - WITH_LIB="urllib31.10" matrix: allow_failures: - env: WITH_LIB="boto" @@ -37,4 +40,7 @@ install: - if [ $WITH_LIB = "requests2.5" ] ; then pip install requests==2.5.0; fi - if [ $WITH_LIB = "httplib2" ] ; then pip install httplib2; fi - if [ $WITH_LIB = "boto" ] ; then pip install boto; fi +- if [ $WITH_LIB = "urllib31.7" ] ; then pip install certifi urllib3==1.7.1; fi +- if [ $WITH_LIB = "urllib31.9" ] ; then pip install certifi urllib3==1.9.1; fi +- if [ $WITH_LIB = "urllib31.10" ] ; then pip install certifi urllib3==1.10.2; fi script: python setup.py test diff --git a/tests/integration/test_urllib3.py b/tests/integration/test_urllib3.py new file mode 100644 index 0000000..017bb3f --- /dev/null +++ b/tests/integration/test_urllib3.py @@ -0,0 +1,153 @@ +'''Integration tests with urllib3''' + +# coding=utf-8 + +import os +import pytest +import vcr +from assertions import ( + assert_cassette_empty, + assert_cassette_has_one_response, + assert_is_json +) +certifi = pytest.importorskip("certifi") +urllib3 = pytest.importorskip("urllib3") + + +@pytest.fixture(params=["https", "http"]) +def scheme(request): + """ + Fixture that returns both http and https + """ + return request.param + + +@pytest.fixture(scope='module') +def verify_pool_mgr(): + return urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', # Force certificate check. + ca_certs=certifi.where() + ) + + +@pytest.fixture(scope='module') +def pool_mgr(): + return urllib3.PoolManager() + + +def test_status_code(scheme, tmpdir, verify_pool_mgr): + '''Ensure that we can read the status code''' + url = scheme + '://httpbin.org/' + with vcr.use_cassette(str(tmpdir.join('atts.yaml'))): + status_code = verify_pool_mgr.request('GET', url).status + + with vcr.use_cassette(str(tmpdir.join('atts.yaml'))): + assert status_code == verify_pool_mgr.request('GET', url).status + + +def test_headers(scheme, tmpdir, verify_pool_mgr): + '''Ensure that we can read the headers back''' + url = scheme + '://httpbin.org/' + with vcr.use_cassette(str(tmpdir.join('headers.yaml'))): + headers = verify_pool_mgr.request('GET', url).headers + + with vcr.use_cassette(str(tmpdir.join('headers.yaml'))): + assert headers == verify_pool_mgr.request('GET', url).headers + + +def test_body(tmpdir, scheme, verify_pool_mgr): + '''Ensure the responses are all identical enough''' + url = scheme + '://httpbin.org/bytes/1024' + with vcr.use_cassette(str(tmpdir.join('body.yaml'))): + content = verify_pool_mgr.request('GET', url).data + + with vcr.use_cassette(str(tmpdir.join('body.yaml'))): + assert content == verify_pool_mgr.request('GET', url).data + + +def test_auth(tmpdir, scheme, verify_pool_mgr): + '''Ensure that we can handle basic auth''' + auth = ('user', 'passwd') + headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(*auth)) + url = scheme + '://httpbin.org/basic-auth/user/passwd' + with vcr.use_cassette(str(tmpdir.join('auth.yaml'))): + one = verify_pool_mgr.request('GET', url, headers=headers) + + with vcr.use_cassette(str(tmpdir.join('auth.yaml'))): + two = verify_pool_mgr.request('GET', url, headers=headers) + assert one.data == two.data + assert one.status == two.status + + +def test_auth_failed(tmpdir, scheme, verify_pool_mgr): + '''Ensure that we can save failed auth statuses''' + auth = ('user', 'wrongwrongwrong') + headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(*auth)) + url = scheme + '://httpbin.org/basic-auth/user/passwd' + with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))) as cass: + # Ensure that this is empty to begin with + assert_cassette_empty(cass) + one = verify_pool_mgr.request('GET', url, headers=headers) + two = verify_pool_mgr.request('GET', url, headers=headers) + assert one.data == two.data + assert one.status == two.status == 401 + + +def test_post(tmpdir, scheme, verify_pool_mgr): + '''Ensure that we can post and cache the results''' + data = {'key1': 'value1', 'key2': 'value2'} + url = scheme + '://httpbin.org/post' + with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))): + req1 = verify_pool_mgr.request('POST', url, data).data + + with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))): + req2 = verify_pool_mgr.request('POST', url, data).data + + assert req1 == req2 + + +def test_redirects(tmpdir, scheme, verify_pool_mgr): + '''Ensure that we can handle redirects''' + url = scheme + '://httpbin.org/redirect-to?url=bytes/1024' + with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))): + content = verify_pool_mgr.request('GET', url).data + + with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))) as cass: + assert content == verify_pool_mgr.request('GET', url).data + # Ensure that we've now cached *two* responses. One for the redirect + # and one for the final fetch + assert len(cass) == 2 + assert cass.play_count == 2 + + +def test_cross_scheme(tmpdir, scheme, verify_pool_mgr): + '''Ensure that requests between schemes are treated separately''' + # First fetch a url under http, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join('cross_scheme.yaml'))) as cass: + verify_pool_mgr.request('GET', 'https://httpbin.org/') + verify_pool_mgr.request('GET', 'http://httpbin.org/') + assert cass.play_count == 0 + assert len(cass) == 2 + + +def test_gzip(tmpdir, scheme, verify_pool_mgr): + ''' + Ensure that requests (actually urllib3) is able to automatically decompress + the response body + ''' + url = scheme + '://httpbin.org/gzip' + response = verify_pool_mgr.request('GET', url) + + with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))): + response = verify_pool_mgr.request('GET', url) + assert_is_json(response.data) + + with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))): + assert_is_json(response.data) + + +def test_https_with_cert_validation_disabled(tmpdir, pool_mgr): + with vcr.use_cassette(str(tmpdir.join('cert_validation_disabled.yaml'))): + pool_mgr.request('GET', 'https://httpbin.org') diff --git a/tox.ini b/tox.ini index 2389a92..064c21e 100644 --- a/tox.ini +++ b/tox.ini @@ -21,5 +21,7 @@ deps = requests23: requests==2.3.0 requests22: requests==2.2.1 httplib2: httplib2 - urllib3: urllib3==1.7.1 + urllib317: urllib3==1.7.1 + urllib319: urllib3==1.9.1 + urllib3110: urllib3==1.10.2 boto: boto diff --git a/vcr/patch.py b/vcr/patch.py index 9408397..cb31947 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -138,39 +138,8 @@ class CassettePatcherBuilder(object): import requests.packages.urllib3.connectionpool as cpool except ImportError: # pragma: no cover return () - from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection - http_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRRequestsHTTPConnection) - ) - https_connection_remover = ConnectionRemover( - self._get_cassette_subclass(VCRRequestsHTTPSConnection) - ) - mock_triples = ( - (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), - (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection), - (cpool, 'HTTPConnection', VCRRequestsHTTPConnection), - (cpool, 'HTTPSConnection', VCRRequestsHTTPSConnection), - (cpool, 'is_connection_dropped', mock.Mock(return_value=False)), # Needed on Windows only - (cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection), - (cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection), - ) - # These handle making sure that sessions only use the - # connections of the appropriate type. - mock_triples += ((cpool.HTTPConnectionPool, '_get_conn', - self._patched_get_conn(cpool.HTTPConnectionPool, - lambda : cpool.HTTPConnection)), - (cpool.HTTPSConnectionPool, '_get_conn', - self._patched_get_conn(cpool.HTTPSConnectionPool, - lambda : cpool.HTTPSConnection)), - (cpool.HTTPConnectionPool, '_new_conn', - self._patched_new_conn(cpool.HTTPConnectionPool, - http_connection_remover)), - (cpool.HTTPSConnectionPool, '_new_conn', - self._patched_new_conn(cpool.HTTPSConnectionPool, - https_connection_remover))) - - return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), - (http_connection_remover, https_connection_remover)) + from .stubs import requests_stubs + return self._mock_urllib3_triples(cpool, requests_stubs) def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @@ -193,17 +162,13 @@ class CassettePatcherBuilder(object): return new_connection return patched_new_conn - @_build_patchers_from_mock_triples_decorator def _urllib3(self): try: import urllib3.connectionpool as cpool except ImportError: # pragma: no cover pass - else: - from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection - - yield cpool, 'VerifiedHTTPSConnection', VCRVerifiedHTTPSConnection - yield cpool, 'HTTPConnection', VCRHTTPConnection + from .stubs import urllib3_stubs + return self._mock_urllib3_triples(cpool, urllib3_stubs) @_build_patchers_from_mock_triples_decorator def _httplib2(self): @@ -229,6 +194,40 @@ class CassettePatcherBuilder(object): else: from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection + + def _mock_urllib3_triples(self, cpool, stubs): + http_connection_remover = ConnectionRemover( + self._get_cassette_subclass(stubs.VCRRequestsHTTPConnection) + ) + https_connection_remover = ConnectionRemover( + self._get_cassette_subclass(stubs.VCRRequestsHTTPSConnection) + ) + mock_triples = ( + (cpool, 'VerifiedHTTPSConnection', stubs.VCRRequestsHTTPSConnection), + (cpool, 'VerifiedHTTPSConnection', stubs.VCRRequestsHTTPSConnection), + (cpool, 'HTTPConnection', stubs.VCRRequestsHTTPConnection), + (cpool, 'HTTPSConnection', stubs.VCRRequestsHTTPSConnection), + (cpool, 'is_connection_dropped', mock.Mock(return_value=False)), # Needed on Windows only + (cpool.HTTPConnectionPool, 'ConnectionCls', stubs.VCRRequestsHTTPConnection), + (cpool.HTTPSConnectionPool, 'ConnectionCls', stubs.VCRRequestsHTTPSConnection), + ) + # These handle making sure that sessions only use the + # connections of the appropriate type. + mock_triples += ((cpool.HTTPConnectionPool, '_get_conn', + self._patched_get_conn(cpool.HTTPConnectionPool, + lambda : cpool.HTTPConnection)), + (cpool.HTTPSConnectionPool, '_get_conn', + self._patched_get_conn(cpool.HTTPSConnectionPool, + lambda : cpool.HTTPSConnection)), + (cpool.HTTPConnectionPool, '_new_conn', + self._patched_new_conn(cpool.HTTPConnectionPool, + http_connection_remover)), + (cpool.HTTPSConnectionPool, '_new_conn', + self._patched_new_conn(cpool.HTTPSConnectionPool, + https_connection_remover))) + + return itertools.chain(self._build_patchers_from_mock_triples(mock_triples), + (http_connection_remover, https_connection_remover)) class ConnectionRemover(object): diff --git a/vcr/stubs/urllib3_stubs.py b/vcr/stubs/urllib3_stubs.py index d91c8f6..214884d 100644 --- a/vcr/stubs/urllib3_stubs.py +++ b/vcr/stubs/urllib3_stubs.py @@ -1,8 +1,13 @@ '''Stubs for urllib3''' -from urllib3.connectionpool import VerifiedHTTPSConnection -from ..stubs import VCRHTTPSConnection +from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection +from ..stubs import VCRHTTPConnection, VCRHTTPSConnection +# urllib3 defines its own HTTPConnection classes. It includes some polyfills +# for newer features missing in older pythons. -class VCRVerifiedHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): +class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): + _baseclass = HTTPConnection + +class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): _baseclass = VerifiedHTTPSConnection From e6b43a0374809a15a0d4c4d050a6da55b37ce5c7 Mon Sep 17 00:00:00 2001 From: aisch Date: Mon, 23 Mar 2015 13:43:30 -0700 Subject: [PATCH 2/3] rename urllib3 patch method and rm unused imports from tests --- tests/integration/test_urllib3.py | 2 -- vcr/patch.py | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_urllib3.py b/tests/integration/test_urllib3.py index 017bb3f..704199b 100644 --- a/tests/integration/test_urllib3.py +++ b/tests/integration/test_urllib3.py @@ -2,12 +2,10 @@ # coding=utf-8 -import os import pytest import vcr from assertions import ( assert_cassette_empty, - assert_cassette_has_one_response, assert_is_json ) certifi = pytest.importorskip("certifi") diff --git a/vcr/patch.py b/vcr/patch.py index cb31947..d52c05b 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -139,7 +139,7 @@ class CassettePatcherBuilder(object): except ImportError: # pragma: no cover return () from .stubs import requests_stubs - return self._mock_urllib3_triples(cpool, requests_stubs) + return self._urllib3_patchers(cpool, requests_stubs) def _patched_get_conn(self, connection_pool_class, connection_class_getter): get_conn = connection_pool_class._get_conn @@ -166,9 +166,9 @@ class CassettePatcherBuilder(object): try: import urllib3.connectionpool as cpool except ImportError: # pragma: no cover - pass + return () from .stubs import urllib3_stubs - return self._mock_urllib3_triples(cpool, urllib3_stubs) + return self._urllib3_patchers(cpool, urllib3_stubs) @_build_patchers_from_mock_triples_decorator def _httplib2(self): @@ -195,7 +195,7 @@ class CassettePatcherBuilder(object): from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection - def _mock_urllib3_triples(self, cpool, stubs): + def _urllib3_patchers(self, cpool, stubs): http_connection_remover = ConnectionRemover( self._get_cassette_subclass(stubs.VCRRequestsHTTPConnection) ) From 8930c97ff777525f1d1e7f582f9ffa7064466214 Mon Sep 17 00:00:00 2001 From: aisch Date: Mon, 23 Mar 2015 13:56:48 -0700 Subject: [PATCH 3/3] rm unused imports --- tests/integration/test_requests.py | 7 +------ tests/integration/test_urllib2.py | 5 +---- tests/integration/test_urllib3.py | 5 +---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py index 9f6484f..67c4b32 100644 --- a/tests/integration/test_requests.py +++ b/tests/integration/test_requests.py @@ -2,14 +2,9 @@ # coding=utf-8 -import os import pytest import vcr -from assertions import ( - assert_cassette_empty, - assert_cassette_has_one_response, - assert_is_json -) +from assertions import assert_cassette_empty, assert_is_json requests = pytest.importorskip("requests") diff --git a/tests/integration/test_urllib2.py b/tests/integration/test_urllib2.py index cd97e9e..e31c174 100644 --- a/tests/integration/test_urllib2.py +++ b/tests/integration/test_urllib2.py @@ -1,9 +1,6 @@ '''Integration tests with urllib2''' # coding=utf-8 -# External imports -import os - import pytest from six.moves.urllib.request import urlopen from six.moves.urllib_parse import urlencode @@ -11,7 +8,7 @@ from six.moves.urllib_parse import urlencode # Internal imports import vcr -from assertions import assert_cassette_empty, assert_cassette_has_one_response +from assertions import assert_cassette_has_one_response @pytest.fixture(params=["https", "http"]) diff --git a/tests/integration/test_urllib3.py b/tests/integration/test_urllib3.py index 704199b..694416f 100644 --- a/tests/integration/test_urllib3.py +++ b/tests/integration/test_urllib3.py @@ -4,10 +4,7 @@ import pytest import vcr -from assertions import ( - assert_cassette_empty, - assert_is_json -) +from assertions import assert_cassette_empty, assert_is_json certifi = pytest.importorskip("certifi") urllib3 = pytest.importorskip("urllib3")