Linux tshoot commands

++ netstat -> list of network sockets
# get list of TCP/UDP sockets in LISTENING state along with corresponding processes
netstat -ltunp

# get list of active TCP/UDP sockets along with corresponding processes, exclude localhost connections
netstat -tunp | egrep -v '127.0.0.1'

# get list of active TCP connections with corresponding processes and timers
netstat -tnp -o

# get detailed network statistic
nestat -s

# get list of network interfaces with brief statistic
netstat -I


++ iptables -> tool for managing embedded firewall nf
# get detailed list of rules in filter(actual firewall) table
iptables -L -v -n

# the same as above, but for NAT table, basically NAT rules
iptables -L -t nat -v -n

# stop iptables, essentially disable firewall
/etc/init.d/iptables stop

# start iptables, enable firewall
/etc/init.d/iptables start

# reset iptables counters to 0, can be useful when troubleshooting connections
iptables -Z


++ arp -> checking arp table
arp -v -n -e


++ ethtool -> tool for managing network cards
# get detailed statistic from interface eth0
ethtool -S eth0

# show features such as LRO/GRO/GSO of network interface
ethtool -k eth0


++ nc -> network swiss-knife
# test TCP connection to remote host 10.0.0.1 on port 8443
nc -v -z 10.0.0.1 8443

# open a UDP "connection" to remote host 10.0.0.1 on port 123
nc -u -v 10.0.0.1 123

# test connection between two hosts on a given port
host1$ nc -l 8500
host2$ nc -v -z host1 8500


++ arping -> tool which can generate arp-requests, useful for layer 2 network troubleshooting
# send broadcast arp-request for ip 10.0.0.1
arping -b 10.0.0.1

# detect duplicate addresses of 10.0.0.1 on your broadcast domain
arping -D 10.0.0.1


++ tcpdump -> packet-capture utility (for more filter expressions see "man pcap-filter")
# sniff packets for TCP port 5060 from/to host 10.0.0.1 on interface eth0 and print each packet in ASCII
tcpdump -nn -i eth0 -A tcp port 5060 and host 10.0.0.1

# the same as above, but write output to pcap file which later can be opened with Wireshark
tcpdump -i eth0 -w /var/log/active/capture1.pcap tcp port 5060 and host 10.229.16.97

# sniff packets for UDP port 123 with verbose protocol decoding
tcpdump -i eth0 -nnvv udp port 123

# read packets from pcap file instead of capturing from interface and display them in verbose, protocol-decoded form
tcpdump -nnvv -r /var/log/active/capture1.pcap


++ openssl -> OpenSSL command line tool
# print certificate which is stored in PEM form(base64 encoded)
openssl x509 -in /path/to/certificate.pem -noout -text

# print certificate which is stored in DER form(binary, ASN-encoded)
openssl x509 -inform DER -in /path/to/certificate.der -noout -text

# don't print all details, print only certain fields such subject/issues/serial/validity
openssl x509 -in /path/to/certificate.pem -noout -serial -subject -issuer -dates

# convert certificate from DER-form to PEM-form, the same can be done vice versa
openssl x509 -inform DER -in /path/to/certificate.der -outform PEM -out /path/to/certificate.pem

# display certificate SHA fingerprint
openssl x509 -in /path/to/certificate.pem -noout -text -fingerprint

# display hash of subject, used to create links in /usr/local/platform/.security/tomcat/trust-certs/
openssl x509 -in /path/to/certificate.pem -noout -hash

# calculate hash of public key in CSR and compare it with hash of public key of signed certificate which was issued based on that CSR, no output means that keys are matching, and certificate was generated from that CSR
diff <(openssl x509 -in ./signed_certificate.pem -noout -modulus | md5sum) <(openssl req -in ./cert_request.csr -noout -modulus | md5sum)

# verify certificate chain, certificate is signed by inter-ca, inter-ca signed by root-ca
openssl verify -CAfile ./root-ca.pem -untrusted ./inter-ca.pem ./signed-cert.pem

# connect to host 10.0.0.1 on port 8443, verify certificate with CA-certificate cacert.pem and print all certificate chain received from server
openssl s_client -CAfile ./cacert.pem -showcerts -connect 10.0.0.1:8443

# connect to host 10.0.0.1 on port 8443 using only TLS 1.2 standard
openssl s_client -tls1_2 -showcerts -connect 10.0.0.1:8443

# the same as above, but not using neither SSL 2.0 neither 3.0
openssl s_client -no_ssl3 -no_ssl2 -showcerts -connect 10.0.0.1:8443

# get list of supported ciphersuits
openssl ciphers | sed 's/:/\n/g'

# the same as above, but get only ciphersuites which incorporate RSA certificate
openssl ciphers | sed 's/:/\n/g' | egrep '\-RSA-'

# connect to host using specific ciphersuit
openssl s_client -cipher ECDHE-RSA-AES128-SHA256 -showcerts -connect 10.0.0.1:8443

# connect to host and decode presented certificate to text instantly
openssl s_client -showcerts -connect 10.0.0.1:8443 | openssl x509 -noout -text


++ find -> find objects in file-system
# find files in directory /var/log/active which filename ends with .log or .txt modified less than 30 hour back
find /var/log/active/ -type f -mmin -30 \( -name '*.log' -o -name '*.txt' \)

# find files in directory /var/log/active which filename ends with .log or .txt modified more than 10 days back
find /var/log/active/ -type f -mtime +10 \( -name '*.log' -o -name '*.txt' \)

# find executable files which contain ccm in it's filename
find /usr/local/ -type f -executable -iname '*ccm*'

# find all .pem certificates in /usr/local/ and print it's file-location and x509 details(if possible)
find /usr/local/ -type f -name '*.pem' -exec echo {} \; -exec openssl x509 -in {} -noout -subject -issuer -serial -dates \; -exec echo "----" \; 2>/dev/null


++ tar -> archive utility
# create archive gzip compressed and add all .log file from current directory to it
tar -czf ./myarchive.tar.gz ./*.log

# unpack tar archive
tar -xzf ./myarchive.tar.gz

# print what is inside archive
tar -tzf ./myarchive.tar.gz

# find .log or .txt files modified less than 30 minutes back in /var/log/active/platform/ directory and add them to archive
find /var/log/active/platform/ -type f -mmin -120 \( -name '*.log' -o -name '*.txt' \) -print0 | tar -czf ./myarchive.tar.gz --null -T -

# the same as previous, but add to archive files only by filenames, without path
/var/log/active/platform/ -type f -mmin -30 \( -name '*.log' -o -name '*.txt' \) -print0 | tar --transform='s,/.*/\(.*\),\1,' -czf ./myarchive.tar.gz --null -T -

# the same as previous, but instead of creating local archive, archive will be transferred on remote server via ssh
find /var/log/active/platform/ -type f -mmin -120 \( -name '*.log' -o -name '*.txt' \) -print0 | tar --transform='s,/.*/\(.*\),\1,' -czf - --null -T - | ssh omozol@10.48.53.219 "cat > ./myarchive.tar.gz"


++ top -> interactive tool for monitoring overall system state
# launch top with refresh rate i
top -d i

# launch top with refresh rate n, exit after screen will be refreshed t times
top -d i -n t

# while running top
shift + p -> sort based on CPU utilization
shift + m -> sort based on memory utilization


++ ps -> get list of processes
# list all processes in long and full form
ps -elf

# list all processes with child-parent relations via ASCII drawing
ps -elf --forest

# list specific process by specifying his PID
ps -lf PID

# list processes started by user with UID
ps -lf -u 500
ps -lf -u root

# list specific process with his LWP(threads) by specifying PID
ps -lfL PID

# get top 10 processes by cpu usage with detailed info
ps -e -o state,user,pid,ppid,%cpu,%mem,rss,vsize,nlwp,time,cmd | sort -rnk5 | head -10

# get top 10 processes by memory usage with detailed info
ps -e -o state,user,pid,ppid,%cpu,%mem,rss,vsize,nlwp,time,cmd | sort -rnk6 | head -10


++ lsof -> get list of file descriptors
# get list of opened file descriptors by process with PID
lsof -p PID -P

# get PID for process which open file file
lsof -n -t /path/to/file

# get list of opened TCP sockets by process with PID
lsof -p PID -P | egrep 'IPv.*TCP'

# list file descriptors for opened files which have following characters in their full path
lsof +D /some/path


++ jobs/bg/fg -> job manipulation
# get list of current jobs
jobs

# put job with ID to foreground execution
fg %ID

# put job with ID to background execution
bg %ID


++ strace -> monitor syscalls of specific process or command, process tracing
# get trace of command in long form with timestamps following child processes and LWP threads
strace -fvvTt -s 256 command

# same as above, but write output to the file for further analysis
strace -fvvTt -s 256 -o /path/to/tracefile.txt command

# same as above but track only specific syscalls, such as open,read,write
strace -fvvTt -s 256 -e open,read,write -o /path/to/tracefile.txt command

# same as above, but instead of launching new command/process, connect to existing process with PID
strace -fvvTt -s 256 -e open,read,write -o /path/to/tracefile.txt -p PID


++ auditctl -> tool for managing kernel audit system
# add a rule to monitor read/write operations on particular file
auditctl -w /path/to/the/file -p rw -k myauditrule1

# remove rule which monitors read/write operations on particular file
auditctl -W /path/to/the/file -p rw -k myauditrule1

# get list of audit rules
auditctl -l

# get audit information about file from audit logs
egrep -a 'filename' /var/log/active/audit/vos/vos-audit*


++ free -> concise output of memory stats
# memory stats in MB
free -m


++ vmstat -> more detailed memory statistic, can be told to execute multiple times to compare outputs dynamically
# get statistic for active virtual memory
vmstat -a -w

Description of output
    FIELD DESCRIPTION FOR VM MODE
       Procs
           r: The number of runnable processes (running or waiting for run time).
           b: The number of processes in uninterruptible sleep.
       Memory
           swpd: the amount of virtual memory used.
           free: the amount of idle memory.
           buff: the amount of memory used as buffers.
           cache: the amount of memory used as cache.
           inact: the amount of inactive memory.  (-a option)
           active: the amount of active memory.  (-a option)
       Swap
           si: Amount of memory swapped in from disk (/s).
           so: Amount of memory swapped to disk (/s).
       IO
           bi: Blocks received from a block device (blocks/s).
           bo: Blocks sent to a block device (blocks/s).
       System
           in: The number of interrupts per second, including the clock.
           cs: The number of context switches per second.
       CPU
           These are percentages of total CPU time.
           us: Time spent running non-kernel code.  (user time, including nice time)
           sy: Time spent running kernel code.  (system time)
           id: Time spent idle.  Prior to Linux 2.5.41, this includes IO-wait time.
           wa: Time spent waiting for IO.  Prior to Linux 2.5.41, included in idle.
           st: Time stolen from a virtual machine.  Prior to Linux 2.6.11, unknown.

# the same as above, but get output 10 times with interval of 1 second, first row in this mode always represent stats from the moment from system startup
vmstat -a -w 1 10

# get detailed statistic for virtual memory usage
vmstat -s


++ iostat -> get detailed I/O statistic
# display detailed statistic for sda disk
iostat -x sda

Description of output
       tps
                     Indicate the number of transfers per second that were issued to the device. A transfer is an I/O request to the device. Multiple logical requests can be combined  into  a  single  I/O
                     request to the device. A transfer is of indeterminate size.
              Blk_read/s (kB_read/s, MB_read/s)
                     Indicate  the  amount of data read from the device expressed in a number of blocks (kilobytes, megabytes) per second. Blocks are equivalent to sectors and therefore have a size of 512
                     bytes.
              Blk_wrtn/s (kB_wrtn/s, MB_wrtn/s)
                     Indicate the amount of data written to the device expressed in a number of blocks (kilobytes, megabytes) per second.
              Blk_read (kB_read, MB_read)
                     The total number of blocks (kilobytes, megabytes) read.
              Blk_wrtn (kB_wrtn, MB_wrtn)
                     The total number of blocks (kilobytes, megabytes) written.
              rrqm/s
                     The number of read requests merged per second that were queued to the device.
              wrqm/s
                     The number of write requests merged per second that were queued to the device.
              r/s
                     The number (after merges) of read requests completed per second for the device.
              w/s
                     The number (after merges) of write requests completed per second for the device.
              rsec/s (rkB/s, rMB/s)
                     The number of sectors (kilobytes, megabytes) read from the device per second.
              wsec/s (wkB/s, wMB/s)
                     The number of sectors (kilobytes, megabytes) written to the device per second.
              avgrq-sz
                     The average size (in sectors) of the requests that were issued to the device.
              avgqu-sz
                     The average queue length of the requests that were issued to the device.
              await
                     The average time (in milliseconds) for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
              r_await
                     The average time (in milliseconds) for read requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
       w_await
                     The average time (in milliseconds) for write requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
              svctm
                     The average service time (in milliseconds) for I/O requests that were issued to the device. Warning! Do not trust this field any more.  This field will be removed in a future  sysstat
                     version.
              %util
                     Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%.

# the same as above, but get output 10 times with interval of 1 second, first row in this mode always represent stats from the moment from system startup
iostat -x sda 1 10


++ iotop -> top for I/O stats, useful for real-time monitoring of systems I/O and determing I/O offenders
# get I/O statistic, print only processes who perform I/O operations, accumulate I/O statistic
iotop -a -o -P

# print I/O statistic only for a processes which run under user tomcat
iotop -a -o -P -u tomcat


++ sar -> get historical statistic of systems resource utilization, similar to Perfmon, files of collected statistic are stored in /var/log/active/sa/, files are retained for last 30 days
# display CPU statistic from file sa24, which correspond of last days with date 24
sar -u -f /var/log/active/sa/sa24

# display detailed memory statistic for the same day
sar -r -f /var/log/active/sa/sa24

# display detailed I/O statistic for the same day
sar -d -f /var/log/active/sa/sa24

# display detailed network statistic for the same day
sar -n DEV -f /var/log/active/sa/sa24