RecyclerView list not getting updated until it is scrolled

public class DiaryUserAdapter extends RecyclerView.Adapter<DiaryUserAdapter.MyViewHolder> {

    List<DiaryUser> diaryUsers;
    Context context;
    MainViewModel mainViewModel;

    public DiaryUserAdapter(Context context){
        this.context = context;
        mainViewModel = new ViewModelProvider((FragmentActivity)context).get(MainViewModel.class);
    }

    @NonNull
    @Override
    public DiaryUserAdapter.MyViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) {
        View view = LayoutInflater.from(context).inflate(R.layout.user_item, parent, false);
        return new MyViewHolder(view);
    }

    @Override
    public void onBindViewHolder(@NonNull DiaryUserAdapter.MyViewHolder holder, int position) {
        DiaryUser user = diaryUsers.get(position);
        holder.tvName.setText(user.name);
        holder.tvEmail.setText(user.email);
        if(user.userId.equals(mainViewModel.getUserDetails().getUserId())){
            holder.viewDelete.setVisibility(View.INVISIBLE);
        }
    }

    @Override
    public int getItemCount() {
        if(diaryUsers == null){
            return 0;
        }
        return diaryUsers.size();
    }

    public void loadList(List<DiaryUser> list){
        diaryUsers = list;
        notifyDataSetChanged();
    }

    public void addItem(DiaryUser user){
        diaryUsers.add(user);
        this.notifyItemInserted(diaryUsers.size()-1);
//        notifyItemRangeChanged(0, diaryUsers.size());
        //this.notifyDataSetChanged();
    }

    public void updateItem(String name, String email, int index){
        DiaryUser user = diaryUsers.get(index);
        if(name != null) user.name = name;
        if(email != null) user.email = email;
        this.notifyItemChanged(index);
//        notifyDataSetChanged();
    }

    public void deleteItem(int index){
        diaryUsers.remove(index);
        this.notifyItemRemoved(index);
    }

    class MyViewHolder extends RecyclerView.ViewHolder {
        TextView tvName, tvEmail;
        View viewEdit, viewDelete;
        public MyViewHolder(View itemView){
            super(itemView);
            tvName = itemView.findViewById(R.id.tv_manage_username);
            tvEmail = itemView.findViewById(R.id.tv_manage_useremail);
            viewEdit = itemView.findViewById(R.id.view_manage_edit);
            viewDelete = itemView.findViewById(R.id.view_manage_delete);
            viewEdit.setOnClickListener(v ->{
                int position = getAdapterPosition();
                if(position != RecyclerView.NO_POSITION){
                    mainViewModel.setUserToEdit(diaryUsers.get(position));
                    mainViewModel.setEditPosition(position);
                }
            });
            viewDelete.setOnClickListener(v -> {
                int position = getAdapterPosition();
                if(position != RecyclerView.NO_POSITION) mainViewModel.setRemovePosition(getAdapterPosition());
            });
        }
    }
}


/************************ Fragment ****************/

import android.app.ProgressDialog;
import android.os.Bundle;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.Toast;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AlertDialog;
import androidx.fragment.app.DialogFragment;
import androidx.fragment.app.Fragment;
import androidx.fragment.app.FragmentTransaction;
import androidx.lifecycle.ViewModelProvider;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import e.allan.mkadvocates.Adapters.DiaryUserAdapter;
import e.allan.mkadvocates.Models.DiaryUser;
import e.allan.mkadvocates.Models.UserDetails;
import e.allan.mkadvocates.MyRetrofit;
import e.allan.mkadvocates.R;
import e.allan.mkadvocates.Utils.SimpleHelpers;
import e.allan.mkadvocates.ViewModels.MainViewModel;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Response;

import static e.allan.Utils.MyConstants.MANAGE_FRAGMENT_TAG;

public class ManageFragment extends Fragment {

    MainViewModel mainViewModel;
    RecyclerView rv;
    DiaryUserAdapter diaryUserAdapter;

    @Override
    public void onCreate(@Nullable Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        mainViewModel = new ViewModelProvider(getActivity()).get(MainViewModel.class);
        diaryUserAdapter = new DiaryUserAdapter(getActivity());
        if(mainViewModel.getUsersList() == null){
            fetchUsers(); //diaryUserAdapter.loadList(fetchedList); is called in the fetchUsers() method once data is fetched
        }else{
            diaryUserAdapter.loadList(mainViewModel.getUsersList());
        }
        initObservers();
    }

    public void initObservers(){
        //on adapter item click
        mainViewModel.getUserToEdit().observe(getActivity(), user ->{
            if(user != null && rv != null){
                displayUserDialog(true);
            }
        });
        //on update dialog submit
        mainViewModel.getUserUpdate().observe(getActivity(), update ->{
            if(update != null && rv != null){
                updateUser(update);
//diaryUserAdapter.updateItem(newName, newEmail, //mainViewModel.getEditPosition()); is called in the //updateUser() method
            }
        });
        mainViewModel.getNewUser().observe(getActivity(), details ->{
            if(details != null && rv != null){
                createUser(details);
//diaryUserAdapter.addItem(response.body()) is called //after user created in backend
            }
        });
        mainViewModel.getRemovePosition().observe(getActivity(), position ->{
            if(position != null && rv != null){
                displayDeleteAlert(position);
// diaryUserAdapter.deleteItem(index) is called in //the displayDeleteAlert(position) method
            }
        });
    }

    @Nullable
    @Override
    public View onCreateView(@NonNull LayoutInflater inflater, @Nullable ViewGroup container, @Nullable Bundle savedInstanceState) {
        View view = inflater.inflate(R.layout.frag_manage, container, false);
        rv = view.findViewById(R.id.rv_manage_users);
        mainViewModel.setAppTitle("M&K Advocates");
        return view;
    }
    @Override
    public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) {
        super.onViewCreated(view, savedInstanceState);
        rv.setLayoutManager(new LinearLayoutManager(getActivity(), RecyclerView.VERTICAL, false));
        rv.setAdapter(diaryUserAdapter);
        View addFab = view.findViewById(R.id.manage_fab);
        addFab.setOnClickListener(v -> displayUserDialog(false));
    }

}