`Want to upload the files into AWS s3 bucket with help of presigned URL.
We have to do the chunk of the file size and with help of multipart need to uplaod the files.
any idea how to chunk the large files and upload the files into s3 bucket.
please help .
using Amazon;
using Amazon.S3;
using Amazon.S3.Model;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace UploadLargeFiles
{
class FinalUpload
{
//this code is with presigned URL code.
static List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>();
static Dictionary<int, string> lstTag = new Dictionary<int, string>();
static async Task Main()
{
string key = "Filename.txt";
DateTime startdt = DateTime.Now;
Console.Write("Upload With presigned URL Started!File Name:" + key + ":" + startdt);
// Set your AWS access key and secret access key
string accessKey = "accessKey";
string secretKey = "secretKey";
// Set the bucket name and key for the file in the S3 bucket
string bucketName = "bucketName";
// Set the local file path of the file to upload
string filePath = "D:\\" + key;
// Set the chunk size (in bytes)
int chunkSize = 5 * 1024 * 1024; // 5MB
// Create an instance of the Amazon S3 client
var s3Client = new AmazonS3Client(accessKey, secretKey, RegionEndpoint.USWest2);
// Initiate the multipart upload
string uploadId = await InitiateMultipartUpload(s3Client, bucketName, key);
// Calculate the total number of chunks
long fileSize = new FileInfo(filePath).Length;
int totalChunks = (int)Math.Ceiling((double)fileSize / chunkSize);
// Generate presigned URLs for each chunk
List<string> presignedUrls = new List<string>();
// Upload each chunk using multipart upload
using (var fileStream = File.OpenRead(filePath))
{
for (int chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++)
{
string presignedUrl = GeneratePresignedUrl(s3Client, bucketName, key, uploadId, chunkIndex + 1);
presignedUrls.Add(presignedUrl);
// Set the chunk range
long startByte = chunkIndex * chunkSize;
long endByte = Math.Min(startByte + chunkSize - 1, fileSize - 1);
long chunkLength = endByte - startByte + 1;
// Create a chunk stream with the appropriate range
var chunkStream = new PartialStream(fileStream, startByte, chunkLength);
// Upload the chunk
await UploadChunk(presignedUrls[chunkIndex], s3Client, bucketName, key, chunkStream, chunkLength, uploadId, chunkIndex + 1);
}
}
// Complete the multipart upload
await CompleteMultipartUpload(s3Client, bucketName, key, uploadId);
DateTime enddate = DateTime.Now;
TimeSpan datetimedifference = (enddate - startdt);
Console.WriteLine("Upload completed." + enddate+"(Time Difference):"+ datetimedifference);
Console.Read();
}
// Generate a presigned URL for initiating the multipart upload
static string GeneratePresignedUrl(AmazonS3Client s3Client, string bucketName, string key, string UploadId, int partnumber)
{
var request = new GetPreSignedUrlRequest
{
BucketName = bucketName,
Key = key,
UploadId = UploadId,
PartNumber = partnumber,
Verb = HttpVerb.PUT,
Expires = DateTime.UtcNow.AddMinutes(5)
};
string presignedUrl = s3Client.GetPreSignedURL(request);
return presignedUrl;
}
// Initiate the multipart upload
static async Task<string> InitiateMultipartUpload(AmazonS3Client s3Client, string bucketName, string key)
{
var request = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = key
};
var response = await s3Client.InitiateMultipartUploadAsync(request);
return response.UploadId;
}
//Upload a chunk using multipart upload
static async Task UploadChunk(string presignedUrl, AmazonS3Client s3Client, string bucketName, string key, Stream chunkStream, long chunkLength, string uploadId, int partNumber)
{
//Option 1
using (var httpClient = new System.Net.Http.HttpClient())
{
// Create an HTTP PUT request with the presigned URL
var putRequest = new System.Net.Http.HttpRequestMessage(System.Net.Http.HttpMethod.Put, presignedUrl)
{
Content = new System.Net.Http.StreamContent(chunkStream)
};
// Set the Content-Length header for the chunk
putRequest.Content.Headers.ContentLength = chunkLength;
// Send the HTTP request and get the response
var response = await httpClient.SendAsync(putRequest);
// Handle the response as needed
response.EnsureSuccessStatusCode();
lstTag.Add(partNumber, response.Headers.ETag.ToString());
}
}
// Complete the multipart upload
static async Task CompleteMultipartUpload(AmazonS3Client s3Client, string bucketName, string key, string uploadId)
{
var request = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = key,
UploadId = uploadId
};
//request.AddPartETags(uploadResponses);
foreach (var part in lstTag)
{
request.PartETags.Add(new PartETag(part.Key, part.Value));
}
var response = await s3Client.CompleteMultipartUploadAsync(request);
Console.WriteLine("File upload completed successfully with completeMultipartUploadResponse:" + response.HttpStatusCode + ":" + DateTime.Now);
}
}
}
//in request you have to pass partnumber so s3client get to know the partnumber //and in response you will get etag values.
using System;
using System.Threading.Tasks;
using Amazon.S3.Model;
using Amazon.S3;
using System.IO;
using Amazon;
using System.Collections.Generic;
namespace UploadLargeFiles
{
class DDWithoutPresignedURL
{
static List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>();
public static string objectKey = "1500MB.zip";
public static string filePath = "D:\\" + objectKey;
static DateTime startdt = DateTime.Now;
static async Task Main()
{
Console.Write("Upload Without presigned URL Started!File Name:" + objectKey + ":" + startdt);
await UploadFile(filePath);
Console.Write("Upload Ended!" + DateTime.Now);
Console.Read();
}
public static async Task UploadFile(string filePath)
{
var fileInfo = new FileInfo(filePath);
long fileSize = fileInfo.Length;
var accessKey = "accessKey";
var secretKey = "secretKey";
var bucketName = "bucketName";
var partSize = 5 * 1024 * 1024;
var partCount = (int)Math.Ceiling((double)fileSize / partSize);
var s3Client = new AmazonS3Client(accessKey, secretKey, RegionEndpoint.USWest2);
//List<string> preSignedURLs = GeneratePreSignedURLs(accessKey, secretKey, bucketName, objectKey, partCount);
var request = new GetPreSignedUrlRequest
{
BucketName = bucketName,
Key = objectKey,
Expires = DateTime.Now.AddMinutes(5),
Verb = HttpVerb.PUT,
//ContentType = "text/plain"
};
var preSignedURL = s3Client.GetPreSignedURL(request);
//preSignedURLs.Add(preSignedURL);
using (var fileStream = File.OpenRead(filePath))
{
//var s3Client = new AmazonS3Client();
//var uploadId = InitiateMultipartUpload(preSignedURL, s3Client);
var uri = new Uri(preSignedURL);
var bucketNames = uri.Host.Split('.')[0];
var key = uri.AbsolutePath.TrimStart('/');
var initiateMultipartUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketNames,
Key = key
};
var initiateMultipartUploadResponse = await s3Client.InitiateMultipartUploadAsync(initiateMultipartUploadRequest);
var uploadId = initiateMultipartUploadResponse.UploadId;
for (int partNumber = 1; partNumber <= partCount; partNumber++)
{
var bytesRead = 0;
var buffer = new byte[partSize];
bytesRead = fileStream.Read(buffer, 0, partSize);
var uploadPartRequest = new UploadPartRequest
{
//BucketName = preSignedURL.BucketName,
BucketName = bucketName,
//Key = preSignedURL.Key,
Key = objectKey,
UploadId = uploadId,
PartNumber = partNumber,
PartSize = bytesRead,
InputStream = new MemoryStream(buffer, 0, bytesRead)
};
var uploadPartResponse = await s3Client.UploadPartAsync(uploadPartRequest);
uploadResponses.Add(uploadPartResponse);
}
var completeMultipartUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadId
};
completeMultipartUploadRequest.AddPartETags(uploadResponses);
#region not Required
//var uploadPartListRequest = new ListPartsRequest
// {
// BucketName = bucketName,
// Key = objectKey,
// UploadId = uploadId
// };
//var uploadPartListResponse = await s3Client.ListPartsAsync(uploadPartListRequest);
//foreach (var part in uploadPartListResponse.Parts)
// {
// completeMultipartUploadRequest.PartETags.Add(new PartETag(part.PartNumber, part.ETag));
// }
#endregion
var completeMultipartUploadResponse = await s3Client.CompleteMultipartUploadAsync(completeMultipartUploadRequest);
Console.WriteLine("File upload completed successfully with completeMultipartUploadResponse:" + completeMultipartUploadResponse.HttpStatusCode + "");
DateTime enddate = DateTime.Now;
TimeSpan datetimedifference = (enddate - startdt);
Console.WriteLine("Upload completed." + enddate + "(Time Difference):" + datetimedifference);
Console.Read();
}
}
}
}