Invalidating custom PageKeyedDataSource makes recycler view jump

Question

I am trying to implement an android paging library with custom PageKeyedDataSource, This data source will query the data from Database and insert Ads randomly on that page.

I implemented the paging, but whenever I scroll past the second page and invalidate the data source, the recycler view jumps back to the end of the second page.

What is the reason for this?

DataSource:

    class ColorsDataSource(
    private val colorsRepository: ColorsRepository
) : PageKeyedDataSource<Int, ColorEntity>() {

    override fun loadInitial(
        params: LoadInitialParams<Int>,
        callback: LoadInitialCallback<Int, ColorEntity>
    ) {
        Timber.i("loadInitial()  offset 0 params.requestedLoadSize $params.requestedLoadSize")
        val resultFromDB = colorsRepository.getColors(0, params.requestedLoadSize)
        // TODO insert Ads here
        callback.onResult(resultFromDB, null, 1)
    }

    override fun loadAfter(params: LoadParams<Int>, callback: LoadCallback<Int, ColorEntity>) {
        val offset = params.key * params.requestedLoadSize
        Timber.i("loadAfter()    offset $offset params.requestedLoadSize $params.requestedLoadSize")
        val resultFromDB = colorsRepository.getColors(
            offset,
            params.requestedLoadSize
        )
        // TODO insert Ads here
        callback.onResult(resultFromDB, params.key + 1)
    }

    override fun loadBefore(params: LoadParams<Int>, callback: LoadCallback<Int, ColorEntity>) {
        // No- Op
    }
}

BoundaryCallback

class ColorsBoundaryCallback(
    private val colorsRepository: ColorsRepository,
    ioExecutor: Executor,
    private val invalidate: () -> Unit
) : PagedList.BoundaryCallback<ColorEntity>() {

    private val helper = PagingRequestHelper(ioExecutor)

    /**
     * Database returned 0 items. We should query the backend for more items.
     */
    @MainThread
    override fun onZeroItemsLoaded() {
        helper.runIfNotRunning(PagingRequestHelper.RequestType.INITIAL) { pagingRequestHelperCallback ->
            Timber.i("onZeroItemsLoaded() ")
            colorsRepository.colorsApiService.getColorsByCall(
                ColorsRepository.getQueryParams(
                    1,
                    ColorViewModel.PAGE_SIZE
                )
            ).enqueue(object : Callback<List<ColorsModel?>?> {
                override fun onFailure(call: Call<List<ColorsModel?>?>, t: Throwable) {
                    handleFailure(t, pagingRequestHelperCallback)
                }

                override fun onResponse(
                    call: Call<List<ColorsModel?>?>,
                    response: Response<List<ColorsModel?>?>
                ) {
                    handleSuccess(response, pagingRequestHelperCallback)
                }
            })
        }
    }

    private fun handleSuccess(
        response: Response<List<ColorsModel?>?>,
        pagingRequestHelperCallback: PagingRequestHelper.Request.Callback
    ) {
        colorsRepository.saveColorsIntoDb(response.body())
        invalidate.invoke()
        Timber.i("onZeroItemsLoaded() with listOfColors")
        pagingRequestHelperCallback.recordSuccess()
    }

    /**
     * User reached to the end of the list.
     */
    @MainThread
    override fun onItemAtEndLoaded(itemAtEnd: ColorEntity) {
        Timber.i("onItemAtEndLoaded() ")
        helper.runIfNotRunning(PagingRequestHelper.RequestType.AFTER) { pagingRequestHelperCallback ->
            val nextPage = itemAtEnd.nextPage?.toInt() ?: 0
            colorsRepository.colorsApiService.getColorsByCall(
                ColorsRepository.getQueryParams(
                    nextPage,
                    ColorViewModel.PAGE_SIZE
                )
            ).enqueue(object : Callback<List<ColorsModel?>?> {
                override fun onFailure(call: Call<List<ColorsModel?>?>, t: Throwable) {
                    handleFailure(t, pagingRequestHelperCallback)
                }

                override fun onResponse(
                    call: Call<List<ColorsModel?>?>,
                    response: Response<List<ColorsModel?>?>
                ) {
                    handleSuccess(response, pagingRequestHelperCallback)
                }

            })
        }
    }

    private fun handleFailure(
        t: Throwable,
        pagingRequestHelperCallback: PagingRequestHelper.Request.Callback
    ) {
        Timber.e(t)
        pagingRequestHelperCallback.recordFailure(t)
    }
}

Adapter's DiffUtil

class DiffUtilCallBack : DiffUtil.ItemCallback<ColorEntity>() {
        override fun areItemsTheSame(oldItem: ColorEntity, newItem: ColorEntity): Boolean {
            return oldItem == newItem
        }

        override fun areContentsTheSame(oldItem: ColorEntity, newItem: ColorEntity): Boolean {
            return oldItem.hexString == newItem.hexString
                    && oldItem.name == newItem.name
                    && oldItem.colorId == newItem.colorId
        }
    }

ViewModel

class ColorViewModel(private val repository: ColorsRepository) : ViewModel() {

    fun getColors(): LiveData<PagedList<ColorEntity>> = postsLiveData

    private var postsLiveData: LiveData<PagedList<ColorEntity>>
    lateinit var dataSourceFactory: DataSource.Factory<Int, ColorEntity>
    lateinit var dataSource: ColorsDataSource

    init {
        val config = PagedList.Config.Builder()
            .setPageSize(PAGE_SIZE)
            .setEnablePlaceholders(false)
            .build()

        val builder = initializedPagedListBuilder(config)
        val contentBoundaryCallBack =
            ColorsBoundaryCallback(repository, Executors.newSingleThreadExecutor()) {
                invalidate()
            }
        builder.setBoundaryCallback(contentBoundaryCallBack)
        postsLiveData = builder.build()
    }

    private fun initializedPagedListBuilder(config: PagedList.Config):
            LivePagedListBuilder<Int, ColorEntity> {

        dataSourceFactory = object : DataSource.Factory<Int, ColorEntity>() {
            override fun create(): DataSource<Int, ColorEntity> {
                dataSource =  ColorsDataSource(repository)
                return dataSource
            }
        }
        return LivePagedListBuilder<Int, ColorEntity>(dataSourceFactory, config)
    }

    private fun invalidate() {
        dataSource.invalidate()
    }

    companion object {
        const val PAGE_SIZE = 8
    }
}

Answer 1

Every time invalidate() is called, the whole list will be considered invalid and built again in its whole, creating a new DataSource instance. It is actually the expected behaviour, but lets see a step by step sequence of what is happening under the hood to understand the problem:

1. A DataSource instance is created, and its loadInitial method is called, with zero items (As there is no data stored yet)
2. BoundaryCallback's onZeroItemsLoaded will be called, so first data will be fetched, stored and finally, it will invalidate the list, so it will be created again.
3. A new DataSource instance will be created, calling its loadInitial again, but this time, as there is already some data, it will retrieve those previously stored items.
4. User will scroll to the list's bottom, so a new page will be tried to be loaded from the DataSource by calling loadAfter, which will retrieve 0 items as there are no more items to be loaded.
5. So onItemAtEndLoaded in BoundaryCallback will be called, fetching the second page, storing the new items and finally invalidating the whole list again.
6. Again, a new DataSource will be created, calling once more its loadInitial, which will only retrieve the first page items.
7. After, once the loadAfter is called again, it will now be able to retrieve the new page items as they have just been added.
8. This will go on for each page.

The problem here can be identified at Step 6.

The thing is that every time we invalidate the DataSource, its loadInitial will only retrieve the first page items. Although having all the other pages items already stored, the new list will not know about their existence until their corresponding loadAfter is called. So after fetching a new page, storing their items and invalidating the list, there will be a moment in which the new list will only be composed by the first page items (as loadInitial will only retrieve those). This new list will be submitted to the Adapter, and so, the RecyclerView will only show the first page items, giving the impression it jumped up to the first item again. However, the reality is that all the other items have been removed as, in theory, they are no longer in the list. After that, once the user scrolls down, the corresponding loadAfter will be called, and the page items will be retrieved again from the stored ones, until a new page with no stored items yet is hit, making it invalidate the whole list again after storing the new items.

So, in order to avoid this, the trick is to make loadInitial not just always retrieve the first page items, but all the already loaded items. This way, once the page is invalidated and the new DataSource's loadInitial is called, the new list will no longer be composed by just the first page items, but by all the already loaded ones, so that they are not removed from the RecyclerView.

To do so, we could keep track of how many pages have already been loaded, so that we can tell to each new DataSources how many of them should be retrieved at loadInitial.

---

A simple solution would consist on creating a class to keep track of the current page:

class PageTracker {
    var currentPage = 0
}

Then, modify the custom DataSource to receive an instance of this class and update it:

class ColorsDataSource(
    private val pageTracker: PageTracker
    private val colorsRepository: ColorsRepository
) : PageKeyedDataSource<Int, ColorEntity>() {

    override fun loadInitial(
        params: LoadInitialParams<Int>,
        callback: LoadInitialCallback<Int, ColorEntity> 
    ) {
        //...
        val alreadyLoadedItems = (pageTracker.currentPage + 1) * params.requestedLoadSize
        val resultFromDB = colorsRepository.getColors(0, alreadyLoadedItems)
        callback.onResult(resultFromDB, null, pageTracker.currentPage + 1)
    }

    override fun loadAfter(params: LoadParams<Int>, callback: LoadCallback<Int, ColorEntity>) {
        pageTracker.currentPage = params.key
        //...
    }

    //...
}

Finally, create an instance of PageTrackerand pass it to each new DataSource instance

dataSourceFactory = object : DataSource.Factory<Int, ColorEntity>() {

    val pageTracker = PageTracker()

    override fun create(): DataSource<Int, ColorEntity> {
        dataSource =  ColorsDataSource(pageTracker, repository)
        return dataSource
    }
}

---

NOTE 1

It is important to note that if it is needed to refresh the whole list again (due to a pull-to-refresh action or anything else), PageTracker instance will be required to be updated back to currentPage = 0 before invalidating the list.

---

NOTE 2

It is also important to note that this approach is usually not required when using Room, as in this case we probably do not need to create our custom DataSource, but instead make the Dao directly return the DataSource.Factory directly from the query. Then, when we fetch new data due to BoundaryCallback calls and store the items, Room will automatically update our list with all the items.

Answer 2

In DiffUtilCallback at areItemsTheSame compare ids instead of references:

override fun areItemsTheSame(oldItem: ColorEntity, newItem: ColorEntity): Boolean 
              = oldItem.db_id == newItem.db_id

In this way the recyclerView will find previous position from ids instead of references.