Add support for calling httplib.send().

This commit changes the whole core internal flow of requests.
Now, requests are actually physically made lazily when a response
is requested.  This allows the entire request to be sent at once.

Otherwise, it would be impossible to compare whether requests have
already been recorded, since httplib.send() allows you to effectively
stream requests over HTTP.

tests/integration/test_requests.py

@@ -84,7 +84,7 @@ def test_post(tmpdir, scheme):
     '''Ensure that we can post and cache the results'''
     data = {'key1': 'value1', 'key2': 'value2'}
     url = scheme + '://httpbin.org/post'
-    with vcr.use_cassette(str(tmpdir.join('redirect.yaml'))) as cass:
+    with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass:
         # Ensure that this is empty to begin with
         assert_cassette_empty(cass)
         req1 = requests.post(url, data).content
@@ -97,7 +97,7 @@ def test_post(tmpdir, scheme):
 def test_redirects(tmpdir, scheme):
     '''Ensure that we can handle redirects'''
     url = scheme + '://httpbin.org/redirect-to?url=bytes/1024'
-    with vcr.use_cassette(str(tmpdir.join('redirect.yaml'))) as cass:
+    with vcr.use_cassette(str(tmpdir.join('requests.yaml'))) as cass:
         # Ensure that this is empty to begin with
         assert_cassette_empty(cass)
         assert requests.get(url).content == requests.get(url).content

tests/integration/test_wild.py

@@ -3,6 +3,8 @@ requests = pytest.importorskip("requests")
 
 import vcr
 
+import httplib
+
 
 def test_domain_redirect():
     '''Ensure that redirects across domains are considered unique'''
@@ -15,3 +17,31 @@ def test_domain_redirect():
         # Ensure that we've now served two responses. One for the original
         # redirect, and a second for the actual fetch
         assert len(cass) == 2
+
+
+def test_flickr_multipart_upload():
+    """
+    The python-flickr-api project does a multipart
+    upload that confuses vcrpy
+    """
+    def _pretend_to_be_flickr_library():
+        content_type, body = "text/plain", "HELLO WORLD"
+        h = httplib.HTTPConnection("httpbin.org")
+        headers = {
+            "Content-Type": content_type,
+            "content-length": str(len(body))
+        }
+        h.request("POST", "/post/", headers=headers)
+        h.send(body)
+        r = h.getresponse()
+        data = r.read()
+        h.close()
+
+    with vcr.use_cassette('fixtures/vcr_cassettes/flickr.json') as cass:
+        _pretend_to_be_flickr_library()
+        assert len(cass) == 1
+
+    with vcr.use_cassette('fixtures/vcr_cassettes/flickr.json') as cass:
+        assert len(cass) == 1
+        _pretend_to_be_flickr_library()
+        assert cass.play_count == 1

vcr/patch.py

@@ -25,7 +25,11 @@ except ImportError:  # pragma: no cover
 
 
 def install(cassette):
-    '''Install a cassette in lieu of actuall fetching'''
+    """
+    Patch all the HTTPConnections references we can find!
+    This replaces the actual HTTPConnection with a VCRHTTPConnection
+    object which knows how to save to / read from cassettes
+    """
     httplib.HTTPConnection = httplib.HTTP._connection_class = VCRHTTPConnection
     httplib.HTTPSConnection = httplib.HTTPS._connection_class = (
         VCRHTTPSConnection)

vcr/stubs/__init__.py

@@ -64,14 +64,79 @@ class VCRConnectionMixin:
             headers=headers or {}
         )
 
-        # Check if we have a cassette set, and if we have a response saved.
-        # If so, there's no need to keep processing and we can bail
-        if self.cassette and self._vcr_request in self.cassette:
-            return
+        # Note: The request may not actually be finished at this point, so
+        # I'm not sending the actual request until getresponse().  This
+        # allows me to compare the entire length of the response to see if it
+        # exists in the cassette.
 
-        # Otherwise, we should submit the request
-        self._baseclass.request(
-            self, method, url, body=body, headers=headers or {})
+    def send(self, data):
+        '''
+        This method is called after request(), to add additional data to the
+        body of the request.  So if that happens, let's just append the data
+        onto the most recent request in the cassette.
+        '''
+        self._vcr_request.body = (self._vcr_request.body or '') + data
+
+    def _send_request(self, method, url, body, headers):
+        """
+        Coppy+pasted from python stdlib 2.6 source because it
+        has a call to self.send() which I have overridden
+        #stdlibproblems #fml
+        """
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            thelen = None
+            try:
+                thelen = str(len(body))
+            except TypeError, te:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                import os
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0:
+                        print "Cannot stat!!"
+
+            if thelen is not None:
+                self.putheader('Content-Length', thelen)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders()
+
+        if body:
+            self._baseclass.send(self, body)
+
+    def _send_output(self, message_body=None):
+        """
+        Copy-and-pasted from httplib, just so I can modify the self.send()
+        calls to call the superclass's send(), since I had to override the
+        send() behavior, since send() is both an external and internal
+        httplib API.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithm.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self._baseclass.send(self, msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self._baseclass.send(self, message_body)
 
     def getresponse(self, _=False):
         '''Retrieve a the response'''
@@ -84,9 +149,22 @@ class VCRConnectionMixin:
             self.cassette.mark_played(self._vcr_request)
             return VCRHTTPResponse(response)
         else:
-            # Otherwise, we made an actual request, and should return the
-            # response we got from the actual connection
-            response = HTTPConnection.getresponse(self)
+            # Otherwise, we should send the request, then get the response
+            # and return it.
+
+            # make the request
+            self._baseclass.request(
+                self,
+                method=self._vcr_request.method,
+                url=self._vcr_request.url,
+                body=self._vcr_request.body,
+                headers=dict(self._vcr_request.headers or {})
+            )
+
+            # get the response
+            response = self._baseclass.getresponse(self)
+
+            # put the response into the cassette
             response = {
                 'status': {
                     'code': response.status,