1

How can I upload large files to MinIO (AWS S3 compatible API) via gRPC service without buffering data?

I have gRPC service with following definition:

service MediaService {
    rpc UploadMedia(stream UploadMediaRequest) returns (UploadMediaResponse);
}

message UploadMediaRequest {
    oneof Data {
        UploadMediaMetadata metadata = 1;
        UploadMediaStream fileStream = 2;
    }
}

message UploadMediaMetadata {
    string bucket = 1;
    string virtialDirectory = 2;
    string fileName = 3;
    string contentType = 4;
    map<string, string> attributes = 6;
}

message UploadMediaStream {
    bytes bytes = 1;
}

And implementation of UploadMedia:

        public override async Task<UploadMediaResponse> UploadMedia(
            IAsyncStreamReader<UploadMediaRequest> requestStream,
            ServerCallContext context)
        {
            UploadMediaMetadata? metadata = null;
            var token = context.CancellationToken;
            var traceId = context.GetHttpContext().TraceIdentifier;

            await using var memoryStream = new MemoryStream();
            await foreach (var req in requestStream.ReadAllAsync(token))
            {
                if (req.DataCase == UploadMediaRequest.DataOneofCase.Metadata)
                {
                    metadata = req.Metadata;
                    _logger.LogTrace("[Req: {TraceId}] Received metadata", traceId);
                }
                else
                {
                    await memoryStream.WriteAsync(req.FileStream.Bytes.Memory, token);
                    _logger.LogTrace("[Req: {TraceId}] Received chunk of bytes", traceId);
                }
            }

            if (metadata == null)
            {
                throw new RpcException(new Status(StatusCode.InvalidArgument, "Not found metadata."));
            }

            memoryStream.Seek(0L, SeekOrigin.Begin);

            var uploadModel = _mapper.Map<UploadModel>(metadata);
            uploadModel.FileStream = memoryStream;

            var file = await _fileService.UploadFile(uploadModel, token);
            await _eventsService.Notify(new MediaUploadedEvent(file.PublicId), token);

            _logger.LogTrace("[Req: {TraceId}] File uploaded", traceId);

            return new UploadMediaResponse { File = _mapper.Map<RpcFileModel>(file) };
        }

At the method I read request stream and write data to MemoryStream. After that I upload file to storage:

        var putObjectArgs = new PutObjectArgs()
                            .WithStreamData(fileStream)
                            .WithObjectSize(fileStream.Length)
                            .WithObject(virtualPath)
                            .WithBucket(bucket)
                            .WithContentType(contentType)
                            .WithHeaders(attributes);

        return _storage.PutObjectAsync(putObjectArgs, token);

I want to upload files without buffering data in Memory. I think I can write bytes from stream to disk and after that create FileStream, but I don't want one more dependency.

0 Answers0