====================================================================
Here are configs that are working vxlan but not working with ceph
====================================================================
root@os004kvm001:~# cat /etc/network/interfaces
# network interface settings; allo-ovsgenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!

auto lo
iface lo inet loopback

allow-hotplug enp4s0
iface enp4s0 inet static
        address 15.4.3.101/8
        gateway 15.1.1.1

allow-ovs vxlan2
iface vxlan2 inet manual
        ovs_type OVSTunnel
        ovs_bridge vmbr1
        ovs_tunnel_type vxlan
        ovs_tunnel_options options:key=flow options:dst_port=4789 options:remote_ip=15.4.3.102

allow-ovs ceph
iface ceph inet static
        address 14.4.3.101/8
        ovs_type OVSIntPort
        ovs_bridge vmbr1
        ovs_options tag=2
        ovs_mtu 1450

allow-ovs vxlan3
iface vxlan3 inet manual
        ovs_type OVSTunnel
        ovs_bridge vmbr1
        ovs_tunnel_type vxlan
        ovs_tunnel_options options:dst_port=4789 options:key=flow options:remote_ip=15.4.3.103

allow-ovs vmbr0
iface vmbr0 inet manual
        ovs_type OVSBridge
        ovs_ports enp4s0

allow-ovs vmbr1
iface vmbr1 inet manual
        ovs_type OVSBridge
        ovs_ports vxlan2 ceph vxlan3

source /etc/network/interfaces.d/*

====================================================================
ceph osds complain they can not hear from osds on other hosts
====================================================================

2021-04-18T20:39:49.801-0500 7fb1566a7700 -1 osd.0 436 heartbeat_check: no reply from 15.4.3.103:6808 osd.2 ever on either front or back, first ping sent 2021-04-18T20:37:31.914970-0500 (oldest deadline 2021-04-18T20:37:51.914970-0500)
2021-04-18T20:39:50.781-0500 7fb1566a7700 -1 osd.0 436 heartbeat_check: no reply from 15.4.3.103:6808 osd.2 ever on either front or back, first ping sent 2021-04-18T20:37:31.914970-0500 (oldest deadline 2021-04-18T20:37:51.914970-0500)

====================================================================
I did a tcpdump and it looks fine
====================================================================

20:39:34.423445 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [.], ack 5695, win 705, options [nop,nop,TS val 1781661915 ecr 2109025841], length 0
20:39:34.423493 IP 15.4.3.103.6808 > 15.4.3.101.55312: Flags [P.], seq 5695:6370, ack 6369, win 702, options [nop,nop,TS val 2109025841 ecr 1781661914], length 675
20:39:34.423499 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [.], ack 6370, win 727, options [nop,nop,TS val 1781661915 ecr 2109025841], length 0
20:39:39.122927 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [.], seq 6369:7817, ack 6370, win 727, options [nop,nop,TS val 1781666614 ecr 2109025841], length 1448
20:39:39.122939 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [P.], seq 7817:8492, ack 6370, win 727, options [nop,nop,TS val 1781666614 ecr 2109025841], length 675
20:39:39.123457 IP 15.4.3.103.6808 > 15.4.3.101.55312: Flags [.], ack 8492, win 735, options [nop,nop,TS val 2109030542 ecr 1781666614], length 0
20:39:39.123791 IP 15.4.3.103.6808 > 15.4.3.101.55312: Flags [.], seq 6370:7818, ack 8492, win 735, options [nop,nop,TS val 2109030542 ecr 1781666614], length 1448
20:39:39.123807 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [.], ack 7818, win 750, options [nop,nop,TS val 1781666615 ecr 2109030542], length 0
20:39:39.123850 IP 15.4.3.103.6808 > 15.4.3.101.55312: Flags [P.], seq 7818:8493, ack 8492, win 735, options [nop,nop,TS val 2109030542 ecr 1781666614], length 675
20:39:39.123858 IP 15.4.3.101.55312 > 15.4.3.103.6808: Flags [.], ack 8493, win 773, options [nop,nop,TS val 1781666615 ecr 2109030542], length 0
^C