update urllib3 patch/stub to be same as used for requests and add tests

.travis.yml

@@ -12,6 +12,9 @@ env:
     - WITH_LIB="requests1.x"
     - WITH_LIB="httplib2"
     - WITH_LIB="boto"
+    - WITH_LIB="urllib31.7"
+    - WITH_LIB="urllib31.9"
+    - WITH_LIB="urllib31.10"
 matrix:
   allow_failures:
     - env: WITH_LIB="boto"
@@ -37,4 +40,7 @@ install:
 - if [ $WITH_LIB = "requests2.5" ] ; then pip install requests==2.5.0; fi
 - if [ $WITH_LIB = "httplib2" ] ; then pip install httplib2; fi
 - if [ $WITH_LIB = "boto" ] ; then pip install boto; fi
+- if [ $WITH_LIB = "urllib31.7" ] ; then pip install certifi urllib3==1.7.1; fi
+- if [ $WITH_LIB = "urllib31.9" ] ; then pip install certifi urllib3==1.9.1; fi
+- if [ $WITH_LIB = "urllib31.10" ] ; then pip install certifi urllib3==1.10.2; fi
 script: python setup.py test

tests/integration/test_urllib3.py

@@ -0,0 +1,153 @@
+'''Integration tests with urllib3'''
+
+# coding=utf-8
+
+import os
+import pytest
+import vcr
+from assertions import (
+    assert_cassette_empty,
+    assert_cassette_has_one_response,
+    assert_is_json
+)
+certifi = pytest.importorskip("certifi")
+urllib3 = pytest.importorskip("urllib3")
+
+
+@pytest.fixture(params=["https", "http"])
+def scheme(request):
+    """
+    Fixture that returns both http and https
+    """
+    return request.param
+
+
+@pytest.fixture(scope='module')
+def verify_pool_mgr():
+    return urllib3.PoolManager(
+        cert_reqs='CERT_REQUIRED', # Force certificate check.
+        ca_certs=certifi.where()
+    )
+
+
+@pytest.fixture(scope='module')
+def pool_mgr():
+    return urllib3.PoolManager()
+
+
+def test_status_code(scheme, tmpdir, verify_pool_mgr):
+    '''Ensure that we can read the status code'''
+    url = scheme + '://httpbin.org/'
+    with vcr.use_cassette(str(tmpdir.join('atts.yaml'))):
+        status_code = verify_pool_mgr.request('GET', url).status
+
+    with vcr.use_cassette(str(tmpdir.join('atts.yaml'))):
+        assert status_code == verify_pool_mgr.request('GET', url).status
+
+
+def test_headers(scheme, tmpdir, verify_pool_mgr):
+    '''Ensure that we can read the headers back'''
+    url = scheme + '://httpbin.org/'
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))):
+        headers = verify_pool_mgr.request('GET', url).headers
+
+    with vcr.use_cassette(str(tmpdir.join('headers.yaml'))):
+        assert headers == verify_pool_mgr.request('GET', url).headers
+
+
+def test_body(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure the responses are all identical enough'''
+    url = scheme + '://httpbin.org/bytes/1024'
+    with vcr.use_cassette(str(tmpdir.join('body.yaml'))):
+        content = verify_pool_mgr.request('GET', url).data
+
+    with vcr.use_cassette(str(tmpdir.join('body.yaml'))):
+        assert content == verify_pool_mgr.request('GET', url).data
+
+
+def test_auth(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure that we can handle basic auth'''
+    auth = ('user', 'passwd')
+    headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(*auth))
+    url = scheme + '://httpbin.org/basic-auth/user/passwd'
+    with vcr.use_cassette(str(tmpdir.join('auth.yaml'))):
+        one = verify_pool_mgr.request('GET', url, headers=headers)
+
+    with vcr.use_cassette(str(tmpdir.join('auth.yaml'))):
+        two = verify_pool_mgr.request('GET', url, headers=headers)
+        assert one.data == two.data
+        assert one.status == two.status
+
+
+def test_auth_failed(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure that we can save failed auth statuses'''
+    auth = ('user', 'wrongwrongwrong')
+    headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(*auth))
+    url = scheme + '://httpbin.org/basic-auth/user/passwd'
+    with vcr.use_cassette(str(tmpdir.join('auth-failed.yaml'))) as cass:
+        # Ensure that this is empty to begin with
+        assert_cassette_empty(cass)
+        one = verify_pool_mgr.request('GET', url, headers=headers)
+        two = verify_pool_mgr.request('GET', url, headers=headers)
+        assert one.data == two.data
+        assert one.status == two.status == 401
+
+
+def test_post(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure that we can post and cache the results'''
+    data = {'key1': 'value1', 'key2': 'value2'}
+    url = scheme + '://httpbin.org/post'
+    with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))):
+        req1 = verify_pool_mgr.request('POST', url, data).data
+
+    with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))):
+        req2 = verify_pool_mgr.request('POST', url, data).data
+
+    assert req1 == req2
+
+
+def test_redirects(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure that we can handle redirects'''
+    url = scheme + '://httpbin.org/redirect-to?url=bytes/1024'
+    with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))):
+        content = verify_pool_mgr.request('GET', url).data
+
+    with vcr.use_cassette(str(tmpdir.join('verify_pool_mgr.yaml'))) as cass:
+        assert content == verify_pool_mgr.request('GET', url).data
+        # Ensure that we've now cached *two* responses. One for the redirect
+        # and one for the final fetch
+        assert len(cass) == 2
+        assert cass.play_count == 2
+
+
+def test_cross_scheme(tmpdir, scheme, verify_pool_mgr):
+    '''Ensure that requests between schemes are treated separately'''
+    # First fetch a url under http, and then again under https and then
+    # ensure that we haven't served anything out of cache, and we have two
+    # requests / response pairs in the cassette
+    with vcr.use_cassette(str(tmpdir.join('cross_scheme.yaml'))) as cass:
+        verify_pool_mgr.request('GET', 'https://httpbin.org/')
+        verify_pool_mgr.request('GET', 'http://httpbin.org/')
+        assert cass.play_count == 0
+        assert len(cass) == 2
+
+
+def test_gzip(tmpdir, scheme, verify_pool_mgr):
+    '''
+    Ensure that requests (actually urllib3) is able to automatically decompress
+    the response body
+    '''
+    url = scheme + '://httpbin.org/gzip'
+    response = verify_pool_mgr.request('GET', url)
+
+    with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))):
+        response = verify_pool_mgr.request('GET', url)
+        assert_is_json(response.data)
+
+    with vcr.use_cassette(str(tmpdir.join('gzip.yaml'))):
+        assert_is_json(response.data)
+
+
+def test_https_with_cert_validation_disabled(tmpdir, pool_mgr):
+    with vcr.use_cassette(str(tmpdir.join('cert_validation_disabled.yaml'))):
+        pool_mgr.request('GET', 'https://httpbin.org')

tox.ini

@@ -21,5 +21,7 @@ deps =
     requests23: requests==2.3.0
     requests22: requests==2.2.1
     httplib2: httplib2
-    urllib3: urllib3==1.7.1
+    urllib317: urllib3==1.7.1
+    urllib319: urllib3==1.9.1
+    urllib3110: urllib3==1.10.2
     boto: boto

vcr/patch.py

@@ -138,39 +138,8 @@ class CassettePatcherBuilder(object):
             import requests.packages.urllib3.connectionpool as cpool
         except ImportError:  # pragma: no cover
             return ()
-        from .stubs.requests_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection
+        from .stubs import requests_stubs
-        http_connection_remover = ConnectionRemover(
+        return self._mock_urllib3_triples(cpool, requests_stubs)
-            self._get_cassette_subclass(VCRRequestsHTTPConnection)
-        )
-        https_connection_remover = ConnectionRemover(
-            self._get_cassette_subclass(VCRRequestsHTTPSConnection)
-        )
-        mock_triples = (
-            (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection),
-            (cpool, 'VerifiedHTTPSConnection', VCRRequestsHTTPSConnection),
-            (cpool, 'HTTPConnection', VCRRequestsHTTPConnection),
-            (cpool, 'HTTPSConnection', VCRRequestsHTTPSConnection),
-            (cpool, 'is_connection_dropped', mock.Mock(return_value=False)),  # Needed on Windows only
-            (cpool.HTTPConnectionPool, 'ConnectionCls', VCRRequestsHTTPConnection),
-            (cpool.HTTPSConnectionPool, 'ConnectionCls', VCRRequestsHTTPSConnection),
-        )
-        # These handle making sure that sessions only use the
-        # connections of the appropriate type.
-        mock_triples += ((cpool.HTTPConnectionPool, '_get_conn',
-                          self._patched_get_conn(cpool.HTTPConnectionPool,
-                                                 lambda : cpool.HTTPConnection)),
-                         (cpool.HTTPSConnectionPool, '_get_conn',
-                          self._patched_get_conn(cpool.HTTPSConnectionPool,
-                                                 lambda : cpool.HTTPSConnection)),
-                         (cpool.HTTPConnectionPool, '_new_conn',
-                          self._patched_new_conn(cpool.HTTPConnectionPool,
-                                                 http_connection_remover)),
-                         (cpool.HTTPSConnectionPool, '_new_conn',
-                          self._patched_new_conn(cpool.HTTPSConnectionPool,
-                                                 https_connection_remover)))
-
-        return itertools.chain(self._build_patchers_from_mock_triples(mock_triples),
-                               (http_connection_remover, https_connection_remover))
 
     def _patched_get_conn(self, connection_pool_class, connection_class_getter):
         get_conn = connection_pool_class._get_conn
@@ -193,17 +162,13 @@ class CassettePatcherBuilder(object):
             return new_connection
         return patched_new_conn
 
-    @_build_patchers_from_mock_triples_decorator
     def _urllib3(self):
         try:
             import urllib3.connectionpool as cpool
         except ImportError:  # pragma: no cover
             pass
-        else:
+        from .stubs import urllib3_stubs
-            from .stubs.urllib3_stubs import VCRVerifiedHTTPSConnection
+        return self._mock_urllib3_triples(cpool, urllib3_stubs)
-
-            yield cpool, 'VerifiedHTTPSConnection', VCRVerifiedHTTPSConnection
-            yield cpool, 'HTTPConnection', VCRHTTPConnection
 
     @_build_patchers_from_mock_triples_decorator
     def _httplib2(self):
@@ -230,6 +195,40 @@ class CassettePatcherBuilder(object):
             from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection
             yield cpool, 'CertValidatingHTTPSConnection', VCRCertValidatingHTTPSConnection
             
+    def _mock_urllib3_triples(self, cpool, stubs):
+        http_connection_remover = ConnectionRemover(
+            self._get_cassette_subclass(stubs.VCRRequestsHTTPConnection)
+        )
+        https_connection_remover = ConnectionRemover(
+            self._get_cassette_subclass(stubs.VCRRequestsHTTPSConnection)
+        )
+        mock_triples = (
+            (cpool, 'VerifiedHTTPSConnection', stubs.VCRRequestsHTTPSConnection),
+            (cpool, 'VerifiedHTTPSConnection', stubs.VCRRequestsHTTPSConnection),
+            (cpool, 'HTTPConnection', stubs.VCRRequestsHTTPConnection),
+            (cpool, 'HTTPSConnection', stubs.VCRRequestsHTTPSConnection),
+            (cpool, 'is_connection_dropped', mock.Mock(return_value=False)),  # Needed on Windows only
+            (cpool.HTTPConnectionPool, 'ConnectionCls', stubs.VCRRequestsHTTPConnection),
+            (cpool.HTTPSConnectionPool, 'ConnectionCls', stubs.VCRRequestsHTTPSConnection),
+        )
+        # These handle making sure that sessions only use the
+        # connections of the appropriate type.
+        mock_triples += ((cpool.HTTPConnectionPool, '_get_conn',
+                          self._patched_get_conn(cpool.HTTPConnectionPool,
+                                                 lambda : cpool.HTTPConnection)),
+                         (cpool.HTTPSConnectionPool, '_get_conn',
+                          self._patched_get_conn(cpool.HTTPSConnectionPool,
+                                                 lambda : cpool.HTTPSConnection)),
+                         (cpool.HTTPConnectionPool, '_new_conn',
+                          self._patched_new_conn(cpool.HTTPConnectionPool,
+                                                 http_connection_remover)),
+                         (cpool.HTTPSConnectionPool, '_new_conn',
+                          self._patched_new_conn(cpool.HTTPSConnectionPool,
+                                                 https_connection_remover)))
+
+        return itertools.chain(self._build_patchers_from_mock_triples(mock_triples),
+                               (http_connection_remover, https_connection_remover))
+
 
 class ConnectionRemover(object):
 

vcr/stubs/urllib3_stubs.py

@@ -1,8 +1,13 @@
 '''Stubs for urllib3'''
 
-from urllib3.connectionpool import VerifiedHTTPSConnection
+from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection
-from ..stubs import VCRHTTPSConnection
+from ..stubs import VCRHTTPConnection, VCRHTTPSConnection
 
+# urllib3 defines its own HTTPConnection classes. It includes some polyfills
+# for newer features missing in older pythons.
 
-class VCRVerifiedHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection):
+class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection):
+    _baseclass = HTTPConnection
+
+class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection):
     _baseclass = VerifiedHTTPSConnection