SELECT COUNT("messages_message"."id")
FROM "messages_message"
WHERE "messages_message"."ignored" = FALSE
  AND (
    "messages_message"."sender_id" = "users_user"."id"
    OR
    "messages_message"."receiver_id" = "users_user"."id"
  )

from django.db.models import Count, OuterRef, Subquery, Q

User.objects.annotate(
    msgs_count=Count(Subquery(
        Message.objects.filter(
            Q(sender_id=OuterRef('pk')) | Q(receiver_id=OuterRef('pk')),
            ignored=False
        )
    ))
)