ExoPlayer in RecyclerView still playing video that shouldn't be playing

Question

I'm working on implementing ExoPlayer in a RecyclerView. I want to only use one instance of ExoPlayer for performance reasons.

Right now, it only plays the first fully visible recyclerview item on screen, and I have that implemented. However, my problem right now is that when I scroll down the recyclerview and call setPlayer(null) for items that should no longer be playing, that video keeps playing regardless (just at a lower frame rate).

Here are the most relevant parts of my recyclerview adapter:

public class PostAdapter extends RecyclerView.Adapter<RecyclerView.ViewHolder> {

    private SimpleExoPlayer exoPlayer;
    private HlsMediaSource.Factory hlsMediaSourceFactory;

    public PostAdapter() {
        this.exoPlayer = new SimpleExoPlayer.Builder(context)
                .build();
        this.hlsMediaSourceFactory = new HlsMediaSource.Factory(CustomMediaSourceFactory.buildMediaSourceFactory());
    }

    public class PostViewHolder extends RecyclerView.ViewHolder {

        // Video is visible
        public void onVideoVisible() {
            // Attach the player
            attachPlayer();
        }

        // Video is hidden
        public void onVideoHidden() {
            // Detach the player
            detachPlayer();
        }

        public void attachPlayer() {
            if (exoPlayer == null) {
                return;
            }

            if (exoPlayer.getCurrentMediaItem() != hlsMediaSource.getMediaItem()) {
                exoPlayer.addListener(new Player.EventListener() {
                    @Override
                    public void onPlaybackStateChanged(int state) {
                        if (state == Player.STATE_READY) {
                            videoPlayerView.setVisibility(View.VISIBLE);
                        }
                    }
                });

                exoPlayer.setVolume(0);
                exoPlayer.setRepeatMode(Player.REPEAT_MODE_ALL);
                videoPlayerView.setPlayer(exoPlayer);

                // Set the media item to be played.
                exoPlayer.setMediaSource(hlsMediaSource);

                // Prepare the player
                exoPlayer.prepare();
            }

            // Play
            exoPlayer.play();
        }

        public void detachPlayer() {
            // This should detach the player and stop the video, but it doesn't
            videoPlayerView.setPlayer(null);
        }

    }

}

And here is the relevant part of where I handle RecyclerView scrolling to figure out which video is in view:

    recyclerView.addOnScrollListener(new RecyclerView.OnScrollListener() {
        @Override
        public void onScrollStateChanged(@NonNull RecyclerView recyclerView, int newState) {
            super.onScrollStateChanged(recyclerView, newState);

            if (newState == RecyclerView.SCROLL_STATE_IDLE) {
                getFirstFullyVisibleVideoPost();
            }
        }
    });

    private void getFirstFullyVisibleVideoPost() {
        LinearLayoutManager layoutManager = (LinearLayoutManager) recyclerView.getLayoutManager();

        if (layoutManager == null) return;

        int firstVisibleItemPosition = layoutManager.findFirstVisibleItemPosition();
        int lastVisibleItemPosition = layoutManager.findLastVisibleItemPosition();

        Rect scrollBounds = new Rect();
        recyclerView.getDrawingRect(scrollBounds);

        boolean foundFirst = false;
        int[] location = new int[2];
        for (int i = firstVisibleItemPosition; i <= lastVisibleItemPosition; i++) {
            RecyclerView.ViewHolder viewHolder = recyclerView.findViewHolderForAdapterPosition(i);

            if (viewHolder instanceof PostAdapter.PostViewHolder) {
                PostAdapter.PostViewHolder postViewHolder = (PostAdapter.PostViewHolder) viewHolder;
                postViewHolder.videoContainer.getLocationInWindow(location);

                if (foundFirst || location[1] < 0 || location[1] > scrollBounds.bottom) {
                    postViewHolder.onVideoHidden();
                } else {
                    foundFirst = true;
                    postViewHolder.onVideoVisible();
                }
            }
        }
    }

What am I doing wrong? Am I not detaching the ExoPlayer properly for the videos that should no longer be playing? Is setPlayer(null) not enough?