For some reason, the code below does not list more than 1000 objects even after implementing the loop to continue listing using the continuation token assignment, which tells ListObjectsV2Request were to continue and stopping only when IsTruncated is true. My bucket has 33,000 objects and even if the prefix and key are empty strings the results always come back with 1000 objects. Can anybody help me understand why I always get 1000 objects? Markers, are for the old version of the Amazon SDK.
''' <summary>
''' List objects in a AWS-S3 Bucket depending on the Prefix and Key passes to it.
''' See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ListingKeysUsingAPIs.html
''' </summary>
''' <param name="myBucketName">The name of the AWS-S3 bucket.</param>
''' <param name="FilePrefix">Act as a organizer. For example `TestFiles\` can be added in front of many files to group them</param>
''' <param name="Key_or_ActualFilename">The actual name of the file with extension.
''' 'For example `ListOfCustomer1.xls` in TestFiles\ListOfCustomer1.xls
''' </param>
''' <remarks>
''' Notes from Amazon documentation:
''' Iterating through multi-page results:
''' As buckets can contain a virtually unlimited number of keys, the complete results of a list query can be extremely large.
''' To manage large result sets, the Amazon S3 API supports pagination to split them into multiple responses.
''' Each list keys response returns a page of up to 1,000 keys with an indicator indicating if the response is truncated.
''' You send a series of list keys requests until you have received all the keys. AWS SDK wrapper libraries provide the same pagination.
''' Function returns List(Of S3Object). Last modified by FGV on 03/20/2022 09:15PM
''' </remarks>
Async Function ListAFileOrFilesInAWS_S3(myBucketName As String, FilePrefix As String, Key_or_ActualFilename As String) As Task(Of List(Of S3Object))
Try
Dim myClient As S3.IAmazonS3 = Get_AWS_S3_Client()
'
If myClient IsNot Nothing Then
Dim SearchKey As String = ""
If String.IsNullOrEmpty(FilePrefix) And String.IsNullOrEmpty(Key_or_ActualFilename) Then
SearchKey = "" ' Search everywhere in the Bucket
ElseIf String.IsNullOrEmpty(FilePrefix) = False And String.IsNullOrEmpty(Key_or_ActualFilename) Then
SearchKey = FilePrefix ' Search for the file key only. more likely will return a file or two with the same unique filename or key part
ElseIf String.IsNullOrEmpty(FilePrefix) And String.IsNullOrEmpty(Key_or_ActualFilename) = False Then
SearchKey = Key_or_ActualFilename ' Search for the prefix only , more likely will return multiple files with the same prefix
ElseIf String.IsNullOrEmpty(FilePrefix) = False And String.IsNullOrEmpty(Key_or_ActualFilename) = False Then
SearchKey = FilePrefix & "\" & Key_or_ActualFilename
End If
'
Dim mylistOfObjectsRequest As ListObjectsV2Request = New ListObjectsV2Request With {
.BucketName = myBucketName,
.Prefix = SearchKey
}
'
' for some reason it does not list more than 1000 objects,
' even after implementing the loop below and the continuation token assignment!
Dim myListOfObjectsResponse As ListObjectsV2Response
Do
myListOfObjectsResponse = Await myClient.ListObjectsV2Async(mylistOfObjectsRequest)
'
'For Each entry As S3Object In myListOfObjectsResponse.S3Objects
' Console.WriteLine("key = {0} size = {1}", entry.Key, entry.Size)
'Next
'Console.WriteLine("Next Continuation Token: {0}", myListOfObjectsResponse.NextContinuationToken)
''
mylistOfObjectsRequest.ContinuationToken = myListOfObjectsResponse.NextContinuationToken
Loop While (myListOfObjectsResponse.IsTruncated)
'
' code below for troubleshooting purpose only
'For Each fileObject As S3Object In myListOfObjectsResponse.S3Objects
' If fileObject.Size > 0 Then ' if file size = 0 then is a folder must probably!
' MsgBox("Key Or File Full Path: " & fileObject.Key & " - Size: " & fileObject.Size.ToString _
' & " Bytes - Last Modified: " & fileObject.LastModified.ToString)
' End If
'Next
'
' using Linq to return the object everywhere in the bucket, but exclude the "folders" (object with size 0) themselves.
Return myListOfObjectsResponse.S3Objects.Where(Function(file) file.Size > 0).ToList()
'
End If
Catch ex As Exception
add_A_LogEntry(ErrorLogFilePath, "Unknown error listing files (object) via AWS-S3 client. Error: " & ex.Message)
End Try
Return Nothing
End Function
