I'm trying to serve large files in response to an HTTP GET request with a Python web server. I've overridden do_GET(self) successfully and everything works great for sending small zip files but with larger files I only write around 137MB into the wfile output stream before I get a broken pipe error as BaseHTTPServer is calling finish before I'm done writing.
self.send_header("Content-Type", "application/octet-stream")
self.end_headers()
f = open(filename, 'rb')
fullyWritten = False
count = 0
while not fullyWritten:
tmp = f.read(32768)
count = count + len(tmp)
print count # always a different number before crash
if not tmp:
fullyWritten = True
else:
self.wfile.write(tmp) # never finishes writing large files
print "wrote to http body"
... ... wrote to http body 137134080 wrote to http body 137166848 wrote to http body 137199616 ----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 43644) Traceback (most recent call last): File "/usr/lib/python2.7/SocketServer.py", line 295, in _handle_request_noblock self.process_request(request, client_address) File "/usr/lib/python2.7/SocketServer.py", line 321, in process_request self.finish_request(request, client_address) File "/usr/lib/python2.7/SocketServer.py", line 334, in finish_request self.RequestHandlerClass(request, client_address, self) File "/usr/lib/python2.7/SocketServer.py", line 651, in __init__ self.finish() File "/usr/lib/python2.7/SocketServer.py", line 710, in finish self.wfile.close() File "/usr/lib/python2.7/socket.py", line 279, in close self.flush() File "/usr/lib/python2.7/socket.py", line 303, in flush self._sock.sendall(view[write_offset:write_offset+buffer_size]) error: [Errno 32] Broken pipe ----------------------------------------
GO code that's calling the end point:
res, err := http.Get(remoteUri) if err != nil { return err } defer res.Body.Close() bodyBytesBuffer := new(bytes.Buffer) _, err = io.Copy(bodyBytesBuffer, res.Body) if err != nil { return err } urlReader := bytes.NewReader(bodyBytesBuffer.Bytes()) var zipReader *zip.Reader zipReader, err = zip.NewReader(urlReader, int64(urlReader.Len()))