https://getfedora.org/en/workstation/download/
sudo systemct enable --now sshd
# Need kernel to match kernel-devel
sudo dnf update kernel
# sudo reboot

# https://elixir.bootlin.com/linux/latest/source
# https://github.com/torvalds/linux/
# https://blog.stgolabs.net/search/label/linux
# BOOT UP
# https://manybutfinite.com/post/cpu-rings-privilege-and-protection/
# https://manybutfinite.com/post/kernel-boot-process/
# https://manybutfinite.com/post/how-computers-boot-up/

# Program memory
# https://manybutfinite.com/post/system-calls/
# https://manybutfinite.com/post/how-the-kernel-manages-your-memory/
# https://manybutfinite.com/post/anatomy-of-a-program-in-memory/

# Syscalls
# https://alex.dzyoba.com/blog/os-interrupts/
# https://manybutfinite.com/post/what-does-an-idle-cpu-do/
# https://manybutfinite.com/post/when-does-your-os-run/

# Stack
# https://manybutfinite.com/post/journey-to-the-stack/
# https://alex.dzyoba.com/blog/redzone/

# All tools
# syscall vs kernel function
# http://brendangregg.com/Perf/linux_perf_tools_full.png
# https://alex.dzyoba.com/blog/kernel-profiling/
# https://alex.dzyoba.com/blog/systemtap/
# https://medium.com/netflix-techblog/linux-performance-analysis-in-60-000-milliseconds-accc10403c55

# Strace
## How strace works
sudo dnf install strace
sudo strace  ping 8.8.8.8 -c 1

# Trace-cmd
## https://jvns.ca/blog/2017/03/19/getting-started-with-ftrace/
## https://alex.dzyoba.com/blog/ftrace/
## https://raw.githubusercontent.com/torvalds/linux/v4.4/Documentation/trace/ftrace.txt
## https://raw.githubusercontent.com/torvalds/linux/v4.4/Documentation/trace/ftrace-design.txt
## tell about ftrace
## Flamescope? https://www.youtube.com/watch?v=03EC8uA30Pw
### 5 key issues and how netflix cloud is architected to solve them
#### 1. autoscaling (spinnaker)
#### 2. bad push (rollback)
#### 3. instance failure (hystrix timeouts)
#### 4. region failure (zuul 2 reroute traffic)
#### 5. overlooked issue (chaos engineering)
## why do root scope analysis?
### high latency
### growth
### upgrades
## methodologies http://www.brendangregg.com/methodology.html
## red (RATE ERRORS DURATION) USE (utilization saturation errors)
## Anti methods
### bad instance anti method(just delete)
## checklist methodology
## zipkin
## http://brendangregg.com/linuxperf.html
## @43:26 from perf to bpf
## @45:48 ftrace/perf/ebpf
## books http://www.brendangregg.com/books.html
http://kernelshark.org/Documentation.html
sudo dnf install trace-cmd kernelshark
sudo trace-cmd record  -l "*ip_*" -l "*icmp*" -p function_graph ping 8.8.8.8 -c 1
kernelshark


# Bcc/ebpf
## why not just pcap/tcpdump https://sdcast.ksdaemon.ru/2017/07/sdcast-58/ Павел Одинцов
## xdp
## https://www.youtube.com/watch?v=ZdVpKx6Wmc8 1 min
## https://www.youtube.com/watch?v=GsMs3n8CB6g 15 min
## https://www.youtube.com/watch?v=bj3qdEDbCD4 45 min Velocity 2017: Performance Analysis Superpowers with Linux eBPF @14:08 fast new way
## https://www.youtube.com/watch?v=4SiWL5tULnQ More on ebpf
## https://en.wikipedia.org/wiki/IPv4
sudo dnf install bcc
sudo /usr/share/bcc/tools/trace -Ilinux/skbuff.h \
    'ip_send_skb(struct net *netx, struct sk_buff *skb) "ip dst: %d %d %d %d", *(skb->data+16), *(skb->data+17), *(skb->data+18), *(skb->data+19)'
# sudo /usr/share/bcc/tools/execsnoop
# ls and echo difference - type hash which commands

# Get pids
## sudo /usr/share/bcc/tools/tcplife
## curl localhost:8080

# Part I
https://www.youtube.com/watch?v=FJW8nGV4jxY
# Part II
https://www.youtube.com/watch?v=zrr2nUln9Kk
# Slides
https://www.slideshare.net/brendangregg/velocity-2015-linux-perf-tools
# Perf labs
https://github.com/brendangregg/perf-labs/tree/master/src

# Perf
## https://perf.wiki.kernel.org/index.php/Tutorial
sudo dnf install git
sudo dnf install perf
sudo dnf install perl-open.noarch
git clone https://github.com/brendangregg/FlameGraph  # or download it from github
cd FlameGraph
sudo perf record -F 9999 -a -g -- git clone https://github.com/brendangregg/FlameGraph
sudo perf script | ./stackcollapse-perf.pl > out.perf-folded
./flamegraph.pl out.perf-folded > perf-kernel.svg
python3 -m http.server
sudo rm -rf FlameGraph/ out.perf-folded perf-kernel.svg perf.data

# Рассказать о стеках
sudo perf top

#sudo perf record -g -a -e skb:kfree_skb
#sudo perf script

# https://github.com/lukego/blog/issues/13
# https://github.com/pavel-odintsov/drop_watch


# ARP
sudo trace-cmd record  -l "*arp*" -l "icmp_rcv" -p function_graph python3 -m http.server
sudo ip -s -s neigh flush all

# Packet Fragmentation/Defragmentation
sudo trace-cmd record   -l "ip_fragment*" -p function_graph python3 -m http.server
ping 192.168.122.1 -s 2000 -c 5

# Forwarding & Local Delivery


# ICMP
ICMP Header
Применение ICMP
Приём и передача ICMP сообщений

# L4
Available L4 Protocols
L4 Protocol Registration
L3 to L4 Delivery: ip_local_deliver_finish


# Use kfree_skb to detect failures
vm_ip=192.168.122.184
curl $vm_ip
sudo /usr/share/bcc/tools/trace -Ilinux/skbuff.h 'kfree_skb(struct sk_buff *skb) (*(skb->data+23)==80) "ip dst: %d %d %d %d %d", *(skb->data+16), *(skb->data+17), *(skb->data+18), *(skb->data+18), *(skb->data+23)'


# https://sdcast.ksdaemon.ru/2019/01/sdcast-97/ Александр Тоболь


# https://www.youtube.com/watch?v=L5msC6pswQs Debugging Linux Issues with eBPF

sudo trace-cmd record  -l "*ip_fragment*" -p function_graph python3 -m http.server