Deleting multiple buckets from multiple regions

Question

I am trying to delete a large quantity of buckets spread out over different regions. I am having difficulty deleting all the objects in the buckets.

This command fails:

var objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucket.BucketName  }).Result;

With error:

The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint

I am guessing this is happening because my client is initialized in a different region.

How can I get objects from a bucket setting region dynamically?

Here is my full code:

var client = new AmazonS3Client(credentials, RegionEndpoint.USEast1);

var buckets = client.ListBucketsAsync().Result;


foreach (var bucket in buckets.Buckets)
{
    if (!keepList.Contains(bucket.BucketName))
    {
        var location = client.GetBucketLocationAsync(new GetBucketLocationRequest() { BucketName = bucket.BucketName }).Result;

        var objects = client.ListObjectsV2Async(new ListObjectsV2Request() { BucketName = bucket.BucketName  }).Result;

        while (true)
        {
            if (objects.IsTruncated)
            {
                objects = client.ListObjectsV2Async(new ListObjectsV2Request() { BucketName = bucket.BucketName, ContinuationToken = objects.NextContinuationToken}).Result; 
            }
            else
            {
                break;
            }
        }


        var response = client.DeleteBucketAsync(new DeleteBucketRequest()
        {
            BucketName = bucket.BucketName,
            BucketRegion = location.Location
        }).Result;
    }
}

There is a `GetBucketLocation()` API call that returns the region. Therefore, you could loop through each bucket, retrieve its region, create an S3 client for that region, then call your Delete code. — John Rotenstein, Jul 30 '20 at 22:53

score 0 · Answer 1 · answered Jul 30 '20 at 18:42

I couldn't find any way around it, I had to statically map string to a new client instance to do the deleting.

public static void DeleteFromRegion(string location, string bucketName, AWSCredentials credentials)
{

    AmazonS3Client client = null;
    ListObjectsResponse objects = null;

    switch (location)
    {
        case "us-west-1":
            client = new AmazonS3Client(credentials, RegionEndpoint.USWest1);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "us-west-2":
            client = new AmazonS3Client(credentials, RegionEndpoint.USWest2);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "":
        case "us-east-1":
            client = new AmazonS3Client(credentials, RegionEndpoint.USEast1);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "us-east-2":
            client = new AmazonS3Client(credentials, RegionEndpoint.USEast2);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "eu-west-1":
            client = new AmazonS3Client(credentials, RegionEndpoint.EUWest1);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "eu-west-2":
            client = new AmazonS3Client(credentials, RegionEndpoint.EUWest2);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "eu-west-3":
            client = new AmazonS3Client(credentials, RegionEndpoint.EUWest3);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "eu-north-1":
            client = new AmazonS3Client(credentials, RegionEndpoint.EUNorth1);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        case "eu-south-1":
            client = new AmazonS3Client(credentials, RegionEndpoint.EUSouth1);
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName }).Result;
            DeleteObjects(client, objects, bucketName);
            break;
        default:
            throw new Exception("Region not recognized");
    }
}

public static void DeleteObjects(AmazonS3Client client, ListObjectsResponse objects, string bucketName)
{

    while (true)
    {
        var kv = new List<KeyVersion>();
        foreach (var o in objects.S3Objects)
        {
            kv.Add(new KeyVersion{ Key = o.Key });
        }

        if (objects.S3Objects.Count == 0) break;

        client.DeleteObjectsAsync(new DeleteObjectsRequest() { BucketName = bucketName, Objects = kv }).Wait();

        if (objects.IsTruncated)
        {
            objects = client.ListObjectsAsync(new ListObjectsRequest() { BucketName = bucketName, Marker = objects.NextMarker }).Result;
        }
        else
        {
            break;
        }
    }


}

You could simplify this by introducing a `Dictionary` which indexes your lookups (the possible values of `location`). That way you're not compromising on duplicated code. — ColinM, Jul 30 '20 at 18:57

Deleting multiple buckets from multiple regions

1 Answers1