From 43f4eb81569323c1be859c1f32da25f1f601fc28 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Thu, 22 Jun 2017 16:49:59 +0100 Subject: [PATCH 01/11] Fix host and port for proxy connections --- vcr/stubs/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 5ab819d..52b57f4 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -18,7 +18,7 @@ log = logging.getLogger(__name__) class VCRFakeSocket(object): """ A socket that doesn't do anything! - Used when playing back casssettes, when there + Used when playing back cassettes, when there is no actual open socket. """ @@ -36,6 +36,16 @@ class VCRFakeSocket(object): """ return 0 # wonder how bad this is.... + def __nonzero__(self): + """This is hacky too. + + urllib3 checks if sock is truthy before calling + set_tunnel (urllib3/connectionpool.py#L592). + If it is true, it never sets the tunnel and this + breaks proxy requests. + """ + return False + def parse_headers(header_list): """ @@ -130,7 +140,7 @@ class VCRConnection(object): """ Returns empty string for the default port and ':port' otherwise """ - port = self.real_connection.port + port = self._tunnel_port or self.real_connection.port default_port = {'https': 443, 'http': 80}[self._protocol] return ':{}'.format(port) if port != default_port else '' @@ -138,7 +148,7 @@ class VCRConnection(object): """Returns request absolute URI""" uri = "{}://{}{}{}".format( self._protocol, - self.real_connection.host, + self._tunnel_host or self.real_connection.host, self._port_postfix(), url, ) @@ -148,7 +158,7 @@ class VCRConnection(object): """Returns request selector url from absolute URI""" prefix = "{}://{}{}".format( self._protocol, - self.real_connection.host, + self._tunnel_host or self.real_connection.host, self._port_postfix(), ) return uri.replace(prefix, '', 1) @@ -313,6 +323,9 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) + self._tunnel_host = None + self._tunnel_port = None + def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 236dc1f4f299d85af8ab38e19c16aac828556f2b Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 12:25:01 +0100 Subject: [PATCH 02/11] Add test for proxies --- tests/integration/test_proxy.py | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/integration/test_proxy.py diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py new file mode 100644 index 0000000..3212b24 --- /dev/null +++ b/tests/integration/test_proxy.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +'''Test using a proxy.''' + +# External imports +import multiprocessing +import SocketServer +import SimpleHTTPServer +import pytest +requests = pytest.importorskip("requests") + +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr + + +class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): + ''' + Simple proxy server. + + (from: http://effbot.org/librarybook/simplehttpserver.htm). + ''' + def do_GET(self): + self.copyfile(urlopen(self.path), self.wfile) + + +@pytest.yield_fixture(scope='session') +def proxy_server(httpbin): + httpd = SocketServer.ForkingTCPServer(('', 0), Proxy) + proxy_process = multiprocessing.Process( + target=httpd.serve_forever, + ) + proxy_process.start() + yield 'http://{}:{}'.format(*httpd.server_address) + proxy_process.terminate() + + +def test_use_proxy(tmpdir, httpbin, proxy_server): + '''Ensure that it works with a proxy.''' + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cass: + requests.get(httpbin.url, proxies={'http': proxy_server}) + requests.get(httpbin.url, proxies={'http': proxy_server}) From fc95e34bd4d260cf8bc4199c91cf3f285bf346ab Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 13:13:02 +0100 Subject: [PATCH 03/11] Determine proxy based on path --- vcr/stubs/__init__.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 52b57f4..6ebe2e8 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -36,16 +36,6 @@ class VCRFakeSocket(object): """ return 0 # wonder how bad this is.... - def __nonzero__(self): - """This is hacky too. - - urllib3 checks if sock is truthy before calling - set_tunnel (urllib3/connectionpool.py#L592). - If it is true, it never sets the tunnel and this - breaks proxy requests. - """ - return False - def parse_headers(header_list): """ @@ -140,15 +130,18 @@ class VCRConnection(object): """ Returns empty string for the default port and ':port' otherwise """ - port = self._tunnel_port or self.real_connection.port + port = self.real_connection.port default_port = {'https': 443, 'http': 80}[self._protocol] return ':{}'.format(port) if port != default_port else '' def _uri(self, url): """Returns request absolute URI""" - uri = "{}://{}{}{}".format( + if not url.startswith('/'): + # Then this must be a proxy request. + return url + uri = "{0}://{1}{2}{3}".format( self._protocol, - self._tunnel_host or self.real_connection.host, + self.real_connection.host, self._port_postfix(), url, ) @@ -158,7 +151,7 @@ class VCRConnection(object): """Returns request selector url from absolute URI""" prefix = "{}://{}{}".format( self._protocol, - self._tunnel_host or self.real_connection.host, + self.real_connection.host, self._port_postfix(), ) return uri.replace(prefix, '', 1) @@ -323,9 +316,6 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) - self._tunnel_host = None - self._tunnel_port = None - def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 365a98bf665e5e9f4be025f47a10d8b3b1451343 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 15:16:23 +0100 Subject: [PATCH 04/11] Fix failing tests --- tests/integration/test_proxy.py | 2 +- vcr/stubs/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 3212b24..ce67e60 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -37,6 +37,6 @@ def proxy_server(httpbin): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' - with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cass: + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): requests.get(httpbin.url, proxies={'http': proxy_server}) requests.get(httpbin.url, proxies={'http': proxy_server}) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 6ebe2e8..763204e 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -136,7 +136,7 @@ class VCRConnection(object): def _uri(self, url): """Returns request absolute URI""" - if not url.startswith('/'): + if url and not url.startswith('/'): # Then this must be a proxy request. return url uri = "{0}://{1}{2}{3}".format( From bed9e520a371a99132e05511f110a141d22d2a7f Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 15:29:36 +0100 Subject: [PATCH 05/11] Fix `socketserver` for Python 3 --- tests/integration/test_proxy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index ce67e60..ab104b1 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -3,11 +3,10 @@ # External imports import multiprocessing -import SocketServer -import SimpleHTTPServer import pytest requests = pytest.importorskip("requests") +from six.moves import socketserver, SimpleHTTPServer from six.moves.urllib.request import urlopen # Internal imports @@ -26,7 +25,7 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): @pytest.yield_fixture(scope='session') def proxy_server(httpbin): - httpd = SocketServer.ForkingTCPServer(('', 0), Proxy) + httpd = socketserver.ForkingTCPServer(('', 0), Proxy) proxy_process = multiprocessing.Process( target=httpd.serve_forever, ) From eb4774a7d2252789f34801d9026e3d3cabd3be03 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Mon, 3 Jul 2017 16:54:29 +0100 Subject: [PATCH 06/11] Only have a sock attribute after connecting --- vcr/stubs/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcr/stubs/__init__.py b/vcr/stubs/__init__.py index 763204e..a23a731 100644 --- a/vcr/stubs/__init__.py +++ b/vcr/stubs/__init__.py @@ -171,6 +171,8 @@ class VCRConnection(object): # allows me to compare the entire length of the response to see if it # exists in the cassette. + self._sock = VCRFakeSocket() + def putrequest(self, method, url, *args, **kwargs): """ httplib gives you more than one way to do it. This is a way @@ -294,11 +296,13 @@ class VCRConnection(object): with force_reset(): return self.real_connection.connect(*args, **kwargs) + self._sock = VCRFakeSocket() + @property def sock(self): if self.real_connection.sock: return self.real_connection.sock - return VCRFakeSocket() + return self._sock @sock.setter def sock(self, value): @@ -316,6 +320,8 @@ class VCRConnection(object): with force_reset(): self.real_connection = self._baseclass(*args, **kwargs) + self._sock = None + def __setattr__(self, name, value): """ We need to define this because any attributes that are set on the From 06dc2190d64e312b3b8285e69a0d50342bc55b46 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Wed, 18 Oct 2017 12:44:52 +0100 Subject: [PATCH 07/11] Fix format string for Python 2.6 --- tests/integration/test_proxy.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index ab104b1..d3c8df1 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -4,7 +4,6 @@ # External imports import multiprocessing import pytest -requests = pytest.importorskip("requests") from six.moves import socketserver, SimpleHTTPServer from six.moves.urllib.request import urlopen @@ -12,6 +11,9 @@ from six.moves.urllib.request import urlopen # Internal imports import vcr +# Conditional imports +requests = pytest.importorskip("requests") + class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' @@ -24,13 +26,13 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): @pytest.yield_fixture(scope='session') -def proxy_server(httpbin): - httpd = socketserver.ForkingTCPServer(('', 0), Proxy) +def proxy_server(): + httpd = socketserver.ThreadingTCPServer(('', 0), Proxy) proxy_process = multiprocessing.Process( target=httpd.serve_forever, ) proxy_process.start() - yield 'http://{}:{}'.format(*httpd.server_address) + yield 'http://{0}:{1}'.format(*httpd.server_address) proxy_process.terminate() From 4b6b5effc744583eb0227d14d0c5e324c50cf074 Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Fri, 3 Nov 2017 11:07:23 +0000 Subject: [PATCH 08/11] Add headers in proxy server response --- tests/integration/test_proxy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index d3c8df1..a1fc003 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -19,10 +19,15 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' Simple proxy server. - (from: http://effbot.org/librarybook/simplehttpserver.htm). + (Inspired by: http://effbot.org/librarybook/simplehttpserver.htm). ''' def do_GET(self): - self.copyfile(urlopen(self.path), self.wfile) + upstream_response = urlopen(self.path) + self.send_response(upstream_response.status, upstream_response.msg) + for header in upstream_response.headers.items(): + self.send_header(*header) + self.end_headers() + self.copyfile(upstream_response, self.wfile) @pytest.yield_fixture(scope='session') From ff7dd06f4730041db3a9b864d6721a59dffaa94a Mon Sep 17 00:00:00 2001 From: Samuel Fekete Date: Fri, 3 Nov 2017 11:39:13 +0000 Subject: [PATCH 09/11] fix proxy for Python 2 --- tests/integration/test_proxy.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index a1fc003..5dfd285 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -23,8 +23,15 @@ class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): ''' def do_GET(self): upstream_response = urlopen(self.path) - self.send_response(upstream_response.status, upstream_response.msg) - for header in upstream_response.headers.items(): + try: + status = upstream_response.status + headers = upstream_response.headers.items() + except AttributeError: + # In Python 2 the response is an addinfourl instance. + status = upstream_response.code + headers = upstream_response.info().items() + self.send_response(status, upstream_response.msg) + for header in headers: self.send_header(*header) self.end_headers() self.copyfile(upstream_response, self.wfile) From 957db22d5c50b560b923184da04e06b63afb5f60 Mon Sep 17 00:00:00 2001 From: Danilo Shiga Date: Tue, 18 Sep 2018 10:11:33 -0300 Subject: [PATCH 10/11] Improve test_use_proxy with cassette headers and play_count assertion --- tests/integration/test_proxy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 5dfd285..4dfcb92 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -51,5 +51,10 @@ def proxy_server(): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): - requests.get(httpbin.url, proxies={'http': proxy_server}) - requests.get(httpbin.url, proxies={'http': proxy_server}) + response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + + with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette: + cassette_response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + + assert cassette_response.headers == response.headers + assert cassette.play_count == 1 From e9d00a5e2aa304815036ff14008ade2fec06a514 Mon Sep 17 00:00:00 2001 From: Luiz Menezes Date: Tue, 18 Sep 2018 11:13:39 -0300 Subject: [PATCH 11/11] Fix test_use_proxy cassette response type --- tests/integration/test_proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index 4dfcb92..9b3c4ad 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -51,10 +51,10 @@ def proxy_server(): def test_use_proxy(tmpdir, httpbin, proxy_server): '''Ensure that it works with a proxy.''' with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))): - response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + response = requests.get(httpbin.url, proxies={'http': proxy_server}) with vcr.use_cassette(str(tmpdir.join('proxy.yaml'))) as cassette: - cassette_response, _ = requests.get(httpbin.url, proxies={'http': proxy_server}) + cassette_response = requests.get(httpbin.url, proxies={'http': proxy_server}) assert cassette_response.headers == response.headers assert cassette.play_count == 1