Limit defines the number of items/records evaluated using only the KeyCondition(If Present) before applying any filters. To solve this problem, as pointed out in earlier solutions, one approach could be to use a GSI, where the filtering condition is part of your GSI key. However, this is fairly restrictive as it's not practical to introduce a new GSI for every access pattern which requires pagination. A more realistic approach could be to query track the Count in the query response and keep querying & appending the next page of results until the aggregated Count satisfies the client-defined Limit. Keep in mind, that you would need to have some custom logic to build the LastEvaluatedKey, which would be required to fetch the subsequent result-page. In Go, this can be achieved in the following way.
func PaginateWithFilters(ctx context.Context, keyCondition string, filteringCondition int, cursor *Cursor) ([]*Records, error) {
var collectiveResult []map[string]types.AttributeValue
var records []*Records
expr, err := buildFilterQueryExpression(keyCondition, filteringCondition)
if err != nil {
return nil, err
}
queryInput := &dynamodb.QueryInput{
ExpressionAttributeNames: expr.Names(),
ExpressionAttributeValues: expr.Values(),
KeyConditionExpression: expr.KeyCondition(),
FilterExpression: expr.Filter(),
TableName: aws.String(yourTableName),
Limit: aws.Int32(cursor.PageLimit),
}
if cursor.LastEvaluatedKey != nil {
queryInput.ExclusiveStartKey = cursor.LastEvaluatedKey
}
paginator := dynamodb.NewQueryPaginator(dbClient, queryInput)
for {
if !paginator.HasMorePages() {
fmt.Println("no more records in the partition")
cursor.LastEvaluatedKey = nil
break
}
singlePage, err := paginator.NextPage(ctx)
if err != nil {
return nil, err
}
pendingItems := int(cursor.PageLimit) - len(collectiveResult)
if int(singlePage.Count) >= pendingItems {
collectiveResult = append(collectiveResult, singlePage.Items[:pendingItems]...)
cursor.LastEvaluatedKey = buildExclusiveStartKey(singlePage.Items[pendingItems-1])
break
}
collectiveResult = append(collectiveResult, singlePage.Items...)
}
err = attributevalue.UnmarshalListOfMaps(collectiveResult, &records)
if err != nil {
return nil, err
}
return records, nil
}
This Medium article discusses pagination with DynamoDB in a bit more depth and includes code snippets in Go to paginate query responses with filters.
