Upload files with @tornado.web.stream_request_body

Question

I'm trying to use @tornado.web.stream_request_body for upload files.

But I have a problem with upload large files. For example, when I upload PDF file larger than 100 MB (https://yadi.sk/i/rzLQ96pk3Tcef6) it loads incorrectly and it doesn't open in viewers.

Code example:

MAX_STREAMED_SIZE = 1024 * 1024 * 1024

@tornado.web.stream_request_body
class UploadHandler(tornado.web.RequestHandler):
    def prepare(self):
       self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)
       self.f = open(os.path.join('files', '12322.pdf'), "w+b")

    def data_received(self, data):
       self.f.write(data)

    def post(self):
       self.f.close()
       print("upload completed")

What can be the reason of the problem?

That code looks correct to me. Is there any message in the logs? How large is the written file when the request is done? Does it get to the "upload completed" message? Are you sure there's only one upload request in progress at a time? — Ben Darnell, Mar 22 '18 at 02:01
There are no messages except "upload completed". The sizes are different. It seems mime types are also different. The written file size: 196,111,931 bytes Original file size: 196,111,535 bytes Yes, I'm using only one stream when uploading the file — Kirill Marks, Mar 22 '18 at 16:11

score 3 · Answer 1 · answered Aug 29 '18 at 04:31

Try this. Work fine with one file upload, no large mem usage

#! /usr/bin/env python
#-* coding: utf-8 -*

# Official packages

# 3rd-party Packages
import tornado.web

# Local Packages

# CONST
MB = 1024 * 1024
GB = 1024 * MB
TB = 1024 * GB
MAX_STREAMED_SIZE = 16*GB

# Class&Function Defination
@tornado.web.stream_request_body
class UploadHandler(tornado.web.RequestHandler):
    def initialize(self):
        self.bytes_read = 0
        self.meta = dict()
        self.receiver = self.get_receiver()

    # def prepare(self):
    """If no stream_request_body"""
    #     self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)

    def data_received(self, chunk):
        self.receiver(chunk)

    def get_receiver(self):
        index = 0
        SEPARATE = b'\r\n'

        def receiver(chunk):
            nonlocal index
            if index == 0:
                index +=1
                split_chunk             = chunk.split(SEPARATE)
                self.meta['boundary']   = SEPARATE + split_chunk[0] + b'--' + SEPARATE
                self.meta['header']     = SEPARATE.join(split_chunk[0:3])
                self.meta['header']     += SEPARATE *2
                self.meta['filename']   = split_chunk[1].split(b'=')[-1].replace(b'"',b'').decode()

                chunk = chunk[len(self.meta['header']):] # Stream掐头
                import os
                self.fp = open(os.path.join('upload',self.meta['filename']), "wb")
                self.fp.write(chunk)
            else:
                self.fp.write(chunk)
        return receiver

    def post(self, *args, **kwargs):
        # Stream去尾
        self.meta['content_length'] = int(self.request.headers.get('Content-Length')) - \
                                      len(self.meta['header']) - \
                                      len(self.meta['boundary'])

        self.fp.seek(self.meta['content_length'], 0)
        self.fp.truncate()
        self.fp.close()
        self.finish('OK')

# Logic
if __name__ == '__main__':
    pass

score 1 · Answer 2 · answered Aug 24 '18 at 07:39

This is my solution == >

Python code:

import tornado.ioloop
import tornado.options
import tornado.web
import tornado.httpserver

from tornado.options import options, define
define("port", default=8888, help="run on the given port", type=int)

class Appliaction(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", HomeHandler),
            (r"/upload", UploadHandler),
        ]
        settings = dict(
            debug = True,
        )
        super(Appliaction, self).__init__(handlers, **settings)

class HomeHandler(tornado.web.RequestHandler):
    def get(self):
        self.render('form.html')

MAX_STREAMED_SIZE = 1024 * 1024 * 1024

@tornado.web.stream_request_body
class UploadHandler(tornado.web.RequestHandler):
    def initialize(self):
        self.bytes_read = 0
        self.data = b''


    def prepare(self):
        self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)

    def data_received(self, chunck):
        self.bytes_read += len(chunck)
        self.data += chunck

    def post(self):
        this_request = self.request
        value = self.data
        with open('file', 'wb') as f:
            f.write(value)


def Main():
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(Appliaction())
    http_server.listen(options.port)
    tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    Main()

Html code :

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Upload</title>
</head>
<body>
    <h1>Upload</h1>
    <form action="/upload" method="post" enctype="multipart/form-data">
        <input type="file" name="file" id="file" />
        <br />
        <input type="submit" value="upload" />
    </form>
</body>
</html>

Upload files with @tornado.web.stream_request_body

2 Answers2

Linked