Merge branch 'master' into unicode-match-on-body

tests/integration/aiohttp_utils.py

@@ -2,23 +2,42 @@
 import asyncio
 
 import aiohttp
+from aiohttp.test_utils import TestClient
 
 
-@asyncio.coroutine
-def aiohttp_request(loop, method, url, output='text', encoding='utf-8', content_type=None, **kwargs):
+async def aiohttp_request(loop, method, url, output='text', encoding='utf-8', content_type=None, **kwargs):
     session = aiohttp.ClientSession(loop=loop)
     response_ctx = session.request(method, url, **kwargs)
 
-    response = yield from response_ctx.__aenter__()
+    response = await response_ctx.__aenter__()
     if output == 'text':
-        content = yield from response.text()
+        content = await response.text()
     elif output == 'json':
         content_type = content_type or 'application/json'
-        content = yield from response.json(encoding=encoding, content_type=content_type)
+        content = await response.json(encoding=encoding, content_type=content_type)
     elif output == 'raw':
-        content = yield from response.read()
+        content = await response.read()
+    elif output == 'stream':
+        content = await response.content.read()
 
     response_ctx._resp.close()
-    yield from session.close()
+    await session.close()
 
     return response, content
+
+
+def aiohttp_app():
+    async def hello(request):
+        return aiohttp.web.Response(text='hello')
+
+    async def json(request):
+        return aiohttp.web.json_response({})
+
+    async def json_empty_body(request):
+        return aiohttp.web.json_response()
+
+    app = aiohttp.web.Application()
+    app.router.add_get('/', hello)
+    app.router.add_get('/json', json)
+    app.router.add_get('/json/empty', json_empty_body)
+    return app

tests/integration/test_aiohttp.py

@@ -5,7 +5,7 @@ asyncio = pytest.importorskip("asyncio")
 aiohttp = pytest.importorskip("aiohttp")
 
 import vcr  # noqa: E402
-from .aiohttp_utils import aiohttp_request  # noqa: E402
+from .aiohttp_utils import aiohttp_app, aiohttp_request  # noqa: E402
 
 
 def run_in_loop(fn):
@@ -93,6 +93,18 @@ def test_binary(tmpdir, scheme):
         assert cassette.play_count == 1
 
 
+def test_stream(tmpdir, scheme):
+    url = scheme + '://httpbin.org/get'
+
+    with vcr.use_cassette(str(tmpdir.join('stream.yaml'))):
+        resp, body = get(url, output='raw')  # Do not use stream here, as the stream is exhausted by vcr
+
+    with vcr.use_cassette(str(tmpdir.join('stream.yaml'))) as cassette:
+        cassette_resp, cassette_body = get(url, output='stream')
+        assert cassette_body == body
+        assert cassette.play_count == 1
+
+
 def test_post(tmpdir, scheme):
     data = {'key1': 'value1', 'key2': 'value2'}
     url = scheme + '://httpbin.org/post'
@@ -154,3 +166,51 @@ def test_params_on_url(tmpdir, scheme):
         assert request.url == url
         assert cassette_response_json == response_json
         assert cassette.play_count == 1
+
+
+def test_aiohttp_test_client(aiohttp_client, tmpdir):
+    loop = asyncio.get_event_loop()
+    app = aiohttp_app()
+    url = '/'
+    client = loop.run_until_complete(aiohttp_client(app))
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))):
+        response = loop.run_until_complete(client.get(url))
+
+    assert response.status == 200
+    response_text = loop.run_until_complete(response.text())
+    assert response_text == 'hello'
+    response_text = loop.run_until_complete(response.text(errors='replace'))
+    assert response_text == 'hello'
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette:
+        response = loop.run_until_complete(client.get(url))
+
+    request = cassette.requests[0]
+    assert request.url == str(client.make_url(url))
+    response_text = loop.run_until_complete(response.text())
+    assert response_text == 'hello'
+    assert cassette.play_count == 1
+
+
+def test_aiohttp_test_client_json(aiohttp_client, tmpdir):
+    loop = asyncio.get_event_loop()
+    app = aiohttp_app()
+    url = '/json/empty'
+    client = loop.run_until_complete(aiohttp_client(app))
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))):
+        response = loop.run_until_complete(client.get(url))
+
+    assert response.status == 200
+    response_json = loop.run_until_complete(response.json())
+    assert response_json is None
+
+    with vcr.use_cassette(str(tmpdir.join('get.yaml'))) as cassette:
+        response = loop.run_until_complete(client.get(url))
+
+    request = cassette.requests[0]
+    assert request.url == str(client.make_url(url))
+    response_json = loop.run_until_complete(response.json())
+    assert response_json is None
+    assert cassette.play_count == 1

tests/integration/test_httplib2.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 '''Integration tests with httplib2'''
 
-# External imports
+import sys
+
 from six.moves.urllib_parse import urlencode
 import pytest
 import pytest_httpbin.certs
 
-# Internal imports
 import vcr
 
 from assertions import assert_cassette_has_one_response
@@ -19,7 +19,12 @@ def http():
     Returns an httplib2 HTTP instance
     with the certificate replaced by the httpbin one.
     """
-    return httplib2.Http(ca_certs=pytest_httpbin.certs.where())
+    kwargs = {
+        'ca_certs': pytest_httpbin.certs.where()
+    }
+    if sys.version_info[:2] in [(2, 7), (3, 7)]:
+        kwargs['disable_ssl_certificate_validation'] = True
+    return httplib2.Http(**kwargs)
 
 
 def test_response_code(tmpdir, httpbin_both):

tests/integration/test_proxy.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+'''Test using a proxy.'''
+
+# External imports
+import multiprocessing
+import pytest
+
+from six.moves import socketserver, SimpleHTTPServer
+from six.moves.urllib.request import urlopen
+
+# Internal imports
+import vcr
+
+# Conditional imports
+requests = pytest.importorskip("requests")
+
+
+class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    '''
+    Simple proxy server.
+
+    (Inspired by: http://effbot.org/librarybook/simplehttpserver.htm).
+    '''
+    def do_GET(self):
+        upstream_response = urlopen(self.path)
+        try:
+            status = upstream_response.status
+            headers = upstream_response.headers.items()
+        except AttributeError:
+            # In Python 2 the response is an addinfourl instance.
+            status = upstream_response.code
+            headers = upstream_response.info().items()
+        self.send_response(status, upstream_response.msg)
+        for header in headers:
+            self.send_header(*header)
+        self.end_headers()
+        self.copyfile(upstream_response, self.wfile)
+
+
+@pytest.yield_fixture(scope='session')
+def proxy_server():
+    httpd = socketserver.ThreadingTCPServer(('', 0), Proxy)
+    proxy_process = multiprocessing.Process(
+        target=httpd.serve_forever,
+    )
+    proxy_process.start()
+    yield 'http://{}:{}'.format(*httpd.server_address)
+    proxy_process.terminate()
+
+
+def test_use_proxy(tmpdir, httpbin, proxy_server):
+    '''Ensure that it works with a proxy.'''
+    with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))):
+        response = requests.get(httpbin.url, proxies={'http': proxy_server})
+
+    with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette:
+        cassette_response = requests.get(httpbin.url, proxies={'http': proxy_server})
+
+    assert cassette_response.headers == response.headers
+    assert cassette.play_count == 1

tests/integration/test_requests.py

@@ -116,10 +116,10 @@ def test_post_chunked_binary(tmpdir, httpbin):
     assert req1 == req2
 
 
-@pytest.mark.xskip('sys.version_info >= (3, 6)', strict=True, raises=ConnectionError)
-@pytest.mark.xskip((3, 5) < sys.version_info < (3, 6) and
-                   platform.python_implementation() == 'CPython',
-                   reason='Fails on CPython 3.5')
+@pytest.mark.skipif('sys.version_info >= (3, 6)', strict=True, raises=ConnectionError)
+@pytest.mark.skipif((3, 5) < sys.version_info < (3, 6) and
+                    platform.python_implementation() == 'CPython',
+                    reason='Fails on CPython 3.5')
 def test_post_chunked_binary_secure(tmpdir, httpbin_secure):
     '''Ensure that we can send chunked binary without breaking while trying to concatenate bytes with str.'''
     data1 = iter([b'data', b'to', b'send'])

tests/integration/test_urllib2.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 '''Integration tests with urllib2'''
 
+import ssl
 from six.moves.urllib.request import urlopen
 from six.moves.urllib_parse import urlencode
 import pytest_httpbin.certs
@@ -12,7 +13,9 @@ from assertions import assert_cassette_has_one_response
 
 
 def urlopen_with_cafile(*args, **kwargs):
-    kwargs['cafile'] = pytest_httpbin.certs.where()
+    context = ssl.create_default_context(cafile=pytest_httpbin.certs.where())
+    context.check_hostname = False
+    kwargs['context'] = context
     try:
         return urlopen(*args, **kwargs)
     except TypeError:

tests/integration/test_urllib3.py

@@ -20,7 +20,7 @@ def verify_pool_mgr():
 
 @pytest.fixture(scope='module')
 def pool_mgr():
-    return urllib3.PoolManager()
+    return urllib3.PoolManager(cert_reqs='CERT_NONE')
 
 
 def test_status_code(httpbin_both, tmpdir, verify_pool_mgr):

tests/integration/test_wild.py

@@ -59,7 +59,7 @@ def test_flickr_multipart_upload(httpbin, tmpdir):
 def test_flickr_should_respond_with_200(tmpdir):
     testfile = str(tmpdir.join('flickr.yml'))
     with vcr.use_cassette(testfile):
-        r = requests.post("http://api.flickr.com/services/upload")
+        r = requests.post("https://api.flickr.com/services/upload", verify=False)
         assert r.status_code == 200