====== Linux Tricks ======
===== Linux Common =====
==== XML beautiful format command ====
cat xxx.xml | xmllint --format -
==== Remount filesystem as read-write ====
mount -o rw,remount /
==== Useful tools to monitor linux ====
iostat
meminfo/free
mpstat
netstat
nmon
pmap
ps/pstree
sar
strace
tcpdump
top
uptime
vmstat
wireshark
==== Use ipref to test network ====
#UDP mode
#Server side
iperf -u -s
#Client side
iperf -u -c 192.168.1.1 -b 100M -t 60
#Using 30 threads with 5Mbps each
iperf -u -c 192.168.1.1 -b 5M -P 30 -t 60
#Test both download and upload bandwidth
iperf -u -c 192.168.1.1 -b 100M -d -t 60
#TCP mode
#Server side
iperf -s
#Client side
iperf -c 192.168.1.1 -t 60
#Using 30 threads
iperf -c 192.168.1.1 -P 30 -t 60
#Test both download and upload bandwidth
iperf -c 192.168.1.1 -d -t 60
==== Add mount --bind info in /etc/fstab ====
/mnt/hdd1/test /home/test/ext none rw,bind 0 0
==== Scan local IPv4 opened ports ====
nc -zv localhost 1-65535 2>&1 | grep -v "failed"
netstat -lnt4 | awk '{print $4}' | cut -f2 -d: | grep -o '[0-9]*' | sort -n
==== Scan opened ports in given subnet ====
nmap -p 22 192.168.1.0/24
==== Get system serial number ====
sudo dmidecode --type 1 | grep 'Serial Number'
==== Hot resize ext2/3/4 partition ====
# Dangerous, following commands will force update partition table!
sudo -i
DISK=/dev/sdb
sfdisk -d $DISK > ./partition_table
total_sectors=`fdisk -l $DISK | grep 'total [0-9]\+ sectors' | sed 's|.*total \([0-9]\+\) sectors.*|\1|'`
last_partition=`fdisk -l $DISK | grep "^$DISK[0-9]\+" | awk '{print $1}'| tail -1`
start_sector=`cat ./partition_table | grep "$last_partition " | sed 's|.*start= *\([0-9]\+\).*|\1|'`
new_size=`expr $total_sectors - $start_sector`
sed "s|\($last_partition .*size= *\)[0-9]\+\(.*\)|\1$new_size\2|" ./partition_table > ./new_partition_table
sfdisk $DISK < ./new_partition_table
resize2fs $last_partition
==== Print HTTP headers ====
wget --server-response --spider {URL}
curl -I {URL}
w3m -dump_head {URL}
==== Calculate MD5 hash for string ====
echo -n 'hello' | md5sum
==== Validate XML format ====
xmllint --noout test.xml
==== Sed usage ====
# Print line 10-20
sed -n '10,20p' {FILE}
==== Print zombie process PID ====
# Kill zombie process
ps -eal | awk '{ if ($2 == "Z") {print $4}}' | sudo kill -9
# Print zombie process's parent PID
ps -eal | awk '{ if ($2 == "Z") {print $5}}' | sort -u
==== Print process tree ====
ps axwef
pstree
==== Sort process by physically resident memory ====
ps -eo "pid,rss,vsz,size,cputime,%cpu,cmd" --width 120 --sort rss,vsz | sort -k2 -rn | head -10
==== Show files opened by given process ====
lsof -p {PID}
==== Split large file and merge back ====
split –b 1G /xxx_large_file /tmp/split_files
cat /tmp/split_files* > /tmp/merged
==== Execute a program every 5 seconds and fresh output ====
watch -n 5 {COMMAND}
==== Update DDNS ====
nsupdate <
==== Package ISO ====
mkisofs -o {OUTPUT_ISO} -Jrv {ISO_VOLUMN_NAME} {INPUT_FOLDER}
==== Package Linux bootable ISO ====
mkisofs -o "{OUTPUT_ISO}" -Jrvb "isolinux/isolinux.bin" -c "isolinux/boot.cat" -no-emul-boot -boot-load-size 4 -boot-info-table -allow-leading-dots -V "Linux_Boot" "{INPUT_FOLDER}"
==== Download webpage recursively ====
wget -r --no-parent -nv -nH --cut-dirs=1 --reject "index.html*" http://xxxx/yyy/zzz -P {LOCAL_DIR}
==== View current network connect and pid ====
sudo netstat -atnp
==== Some useful bash alias ====
alias wget="wget --no-check-certificate"
alias grep="grep -i --color=auto -s"
alias ssh="ssh -o StrictHostKeyChecking=no"
alias scp="scp -o StrictHostKeyChecking=no"
==== Generate random string ====
openssl rand -hex 8 2>/dev/null
< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-16}; echo;
==== Add swap file ====
sudo dd if=/dev/zero of=/swapfile bs=1024 count=512k
sudo mkswap /swapfile
sudo swapon /swapfile
swapon -s
sudo bash -c "cat >> /etc/fstab" <
==== Mount root in r/w mode ====
mount -o remount,rw /
==== SSH auto login ====
# ssh-keygen -t rsa -b 2048
cat ~/.ssh/id_rsa.pub | ssh user@server "cat >> ~/.ssh/authorized_keys"
==== Copy a disk over SSH ====
ssh root@{HOST} "dd if=/dev/sdb bs=1k conv=sync,noerror | gzip -c" | gunzip -c | dd of=/dev/sdb bs=1M
==== Bypass ssh/scp host verification ====
ssh -o StrictHostKeyChecking=no user@host
scp -o StrictHostKeyChecking=no ...
or
cat >> ~/.ssh/config <
==== Small tricks in linux shell ====
sudo !! #Use sudo to execute last command
python -m SimpleHTTPServer #A small web server in Python
:w !sudo tee % #Vim command to force write file with permissions
cd - #Cd to last directory
^foo^bar #Replace foo with bar in last command and then execute it
cp filename{,.bak} #Shortcut for backup file
mtr IP_address #Test network
!whatever:p #Print history commands contains whatever without execute it
cal #Show calendar
${variable:0:5} #Cut string variable
stat -c %U xxx #Get file owner
cmd #Skip bash history, note the heading space
printf '%100s\n' | tr ' ' - #Pring a line using '-'
vim + {FILE} #Open file and jump to bottom
==== Bypass sudo password ====
sudo bash -c "cat >> /etc/sudoers" <
==== Add a Linux user account without password ====
adduser --disabled-password --no-create-home --gecos "" {username}
==== Clear disk cache ====
sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches"
==== Turn off filesystem check ====
/sbin/tune2fs -c 0 -i 0 /dev/{device_name}
==== Test harddisk speed ====
# Test cache speed
sudo hdparm -Tt /dev/sda
# Test continuous large file write speed
time dd if=/dev/zero of=/tmp/2G_tmp bs=1M count=2kB conv=fdatasync
# Test continuous large file read speed
time dd if=/tmp/2G_tmp bs=1M count=2kB | dd of=/dev/null 2>/dev/null
rm -rf /tmp/2G_tmp
# Test sequential/random read/write speed from 4k to 16k
iozone -Rab iozone-result_speed.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k
# Test sequential/random read/write IOPS from 4k to 16k
iozone -Rab iozone-result_iops.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k -O
# Use bonnie++ to test disk performance
sudo bonnie++ -u root
==== Get current disk IO load every 5 seconds ====
sudo iostat -x 5
==== Scan hot-added SCSI disk ====
echo "- - -" > /sys/class/scsi_host/host{scsi_host_id}/scan
==== Refresh SCSI disk size ====
echo 1 > /sys/block/{device_name}/device/rescan
==== Convert Linux text file to Windows style ====
sed -i 's/$/\r/' xxx.txt
==== Remove ^M in text file ====
cat xxx.txt | tr -d "^M" > xxx_new.txt
sed -i "s/^M//g" xxx.txt
==== Grep a file to remove blank and comment (#) lines ====
grep -v "^#\|^$"
==== Remove dead symbolic links ====
find -L -type l -delete
#Avoid follow the link recursively
find -L -maxdepth 1 -type l -delete
==== Delete empty directory recursively ====
find ./ -depth -empty -type d -exec rmdir -v {} \;
==== Find top 10 biggest files in current directory recursively ====
find . -printf '%s %p\n'| sort -nr | head -10
==== Show physical memory slot information ====
sudo lshw
sudo dmidecode -t memory
==== List all PCI devices on bus tree ====
lspci -tv
==== List all USB devices on bus tree ====
lsusb -tv
==== List all block device====
lsblk
==== Test whether can establish TCP connection ====
timeout 1 bash -c 'echo > /dev/tcp/10.110.160.222/22 && echo "accessible"' || echo "not accessible"
nc -z -w 1 10.110.160.2 22 && echo "accessible" || echo "not accessible"
==== Configure LVM partition ====
#sudo apt-get install lvm2
sudo pvcreate /dev/sda6 /dev/sda7
#Check physical volumes
sudo pvscan
sudo pvdisplay
sudo vgcreate vol_grp1 /dev/sda6 /dev/sda7
sudo vgdisplay
#sudo lvcreate -L 20G vg1
# Use all the free space to create logical volume
sudo lvcreate –l 100%FREE –n logical_vol1 vol_grp1
#Extend logical volume to 100GB
sudo lvextend -L100G /dev/vol_grp1/logical_vol1
==== Lookup distro information ====
lsb_release -a
cat /etc/*release
==== Dump HTTP POST/GET request using tcpdump ====
# 0x504f5354 is ASCII code for POST, change 0xdeadbeaf to match url accordingly.
tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x504f5354) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 5):4] = 0xdeadbeaf)' -w ~/post.log -U -A
# 0x47455420 is ASCII code for GET, change 0xdeadbeaf to match url accordingly.
tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 4):4] = 0xdeadbeaf)' -w ~/get.log -U -A
# Source: http://staff.washington.edu/dittrich/talks/core02/tools/tcpdump-filters.txt
NOTES ABOUT TCPDUMP FILTERS
===========================
Expression Meaning
========== =======
[x:y] start at offset x from the beginning of packet and read y bytes
[x] abbreviation for [x:1]
proto[x:y] start at offset x into the proto header and read y bytes
p[x:y] & z = 0 p[x:y] has none of the bits selected by z
p[x:y] & z != 0 p[x:y] has any of the bits selected by z
p[x:y] & z = z p[x:y] has all of the bits selected by z
p[x:y] = z p[x:y] has only the bits selected by z
the usual rules about operator precedence apply; nesting things inside brackets
is probably a good plan. you'll probably want to put the filter into a file or
at least single-quote it on the commandline to stop the shell from interpreting
the metacharacters. !([:])&
Interesting Parts of a Packet
=============================
ip[0] & 0xf0 high nibble: IP version. almost always 4
ip[0] & 0x0f low nibble: header length in 4octet words. should be 5
ip[1] type of service/QoS/DiffServ
ip[2:2] total length of datagram in octets
ip[4:2] IP ID number
ip[6] & 0x80 reserved bit (possibly used for ECN)
ip[6] & 0x40 DF bit
ip[6] & 0x20 MF bit
ip[6:2] & 0x1fff fragment offset (number of 8octet blocks)
ip[8] ttl
ip[9] protocol
ip[10:2] header checksum
ip[12:4] source IP
ip[16:4] destination IP
ip[20..60] there better not be any options in here...
Interesting Parts of an ICMP Message
====================================
icmp[0] type
icmp[1] code
icmp[2:2] checksum
icmp[4...] payload
Interesting Parts of a UDP header
=================================
udp[0:2] source port
udp[2:2] destination port
udp[4:2] datagram length
udp[6:2] UDP checksum
Interesting Parts of a TCP header
=================================
tcp[0:2] source port
tcp[2:2] destination port
tcp[4:4] sequence number
tcp[8:4] acknowledgement number
tcp[12] header length
tcp[13] tcp flags
tcp[14:2] window size
tcp[16:2] checksum
tcp[18:2] urgent pointer
tcp[20..60] options or data
Flags Numerically Meaning
===== =========== =======
---- --S- 0000 0010 = 0x02 normal syn
---A --S- 0001 0010 = 0x12 normal syn-ack
---A ---- 0001 0000 = 0x10 normal ack
--UA P--- 0011 1000 = 0x38 psh-urg-ack. interactive stuff like ssh
---A -R-- 0001 0100 = 0x14 rst-ack. it happens.
---- --SF 0000 0011 = 0x03 syn-fin scan
--U- P--F 0010 1001 = 0x29 urg-psh-fin. nmap fingerprint packet
-Y-- ---- 0100 0000 = 0x40 anything >= 0x40 has a reserved bit set
XY-- ---- 1100 0000 = 0xC0 both reserved bits set
XYUA PRSF 1111 1111 = 0xFF FULL_XMAS scan
ICMP Types and Codes
====================
0 ECHOREPLY
3 UNREACHABLE
3:0 NET
3:1 HOST
3:2 PROTOCOL
3:3 PORT
3:4 NEEDFRAG
3:5 SRC_ROUTE_FAILED
3:6 NET_UNKNOWN
3:7 HOST_UNKNOWN
3:8 SRC_HOST_ISOLATED
3:9 NET_PROHIB
3:10 HOST_PROHIB
3:11 BAD_TOS_FOR_NET
3:12 BAD_TOS_FOR_HOST
3:13 FILTER_PROHIB
3:14 HOST_PRECEDENCE_VIOLATION
3:15 PRECEDENCE_CUTOFF
4 SOURCEQUENCH
5 REDIRECT
5:0 NET
5:1 HOST
5:2 TOSNET
5:3 TOSHOST
8 ECHO
9 ROUTERADVERT
10 ROUTERSOLICIT
11 TIME_EXCEEDED
11:0 IN_TRANSIT
11:1 DURING_FRAG_REASSEMBLY
12 PARAMETER_PROBLEM
12:1 MISSING_OPT_FOR_REQUEST
13 TSTAMP_REQ
14 TSTAMP_REPLY
15 INFO_REQ
16 INFO_REPLY
17 NETMASK_REQ
18 NETMASK_REPLY
Examples
--------
is SYN. nothing else.
tcp[13] = 0x02
contains SYN. we don't care what else...
(tcp[13] & 0x02) != 0
is some kind of SYN-FIN. Bad news
(tcp[13] & 0x03) = 3
land attack
ip[12:4] = ip[16:4]
winnuke
(tcp[2:2] = 139) && (tcp[13] & 0x20 != 0) && (tcp[19] & 0x01 = 1)
things other than ACK/PSH
(tcp[13] & 0xe7) != 0
initial fragments
(ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff = 0)
intervening fragments
(ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff != 0)
terminal fragments
(ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0)
has ip options (or is truncated, or is just some sort of freak...)
(ip[0] & 0x0f) != 5
ping o' death and its ilk (any oversized IP-transported data...)
((ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0)) && \
((65535 < (ip[2:2] + 8*(ip[6:2] & 0x1fff))
===== Ubuntu/Debian =====
==== Automatically install dependencies with "dpkg -i" ====
- dpkg -i
- If it needs dependencies, get them automatically with apt-get -y -f --force-yes install. dpkg -i has generated the list of unresolved dependencies, apt-get -f will just pick it up.
- Run the dpkg -i again
==== Ubuntu/Debian auto login ====
sudo apt-get install mingetty
vim nano /etc/inittab
#change the line 1:2345:respawn:/sbin/getty 38400 tty1 to 1:2345:respawn:/sbin/mingetty --autologin --noclear tty1
==== Connect PPTP VPN ====
pptpsetup --create hackerzhou.me --server {SERVER_IP} --username {USERNAME} --password {PASSWORD} --encrypt --start
==== Install TrueType fonts ====
sudo apt-get install ttf-mscorefonts-installer
==== Install KDE ====
[[http://wiki.ubuntu.org.cn/KDE]]
==== Check rootkit and security holes ====
sudo apt-get install rkhunter
rkhunter --checkall
==== Setup Samba with management UI on Ubuntu ====
sudo apt-get install samba samba-common
sudo apt-get install system-config-samba
sudo smbpasswd -a xxxxx
==== Setup MySQL for Python on Ubuntu ====
sudo apt-get install python-pip python-dev libmysqlclient-dev
pip install -U pip
pip install MySQL-python
==== Setup VLAN on Ubuntu ====
# download latest vlan deb from http://www.ubuntuupdates.org/vlan
sudo dpkg -i vlan_1.9-3ubuntu10_amd64.deb
sudo su -c 'echo "8021q" >> /etc/modules'
sudo bash -c "cat >> /etc/network/interfaces" <
==== Install & config pptpd ====
sudo apt-get install pptpd
sudo bash -c "cat>> /etc/pptpd.conf" <> /etc/ppp/pptpd-options" <> /etc/ppp/chap-secrets " <
==== Install Node.js ====
sudo apt-get install python-software-properties python g++ make
sudo add-apt-repository ppa:chris-lea/node.js
sudo apt-get update
sudo apt-get install nodejs
==== Set timezone to GMT+8 ====
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
or
sudo dpkg-reconfigure tzdata
==== Quick setup VPS ====
apt-get install axel p7zip-full unzip mysql-server apache2 libapache2-mod-fastcgi php5 php5-fpm php5-cgi php5-mysql phpmyadmin
a2dismod autoindex cgi negotiation php5 reqtimeout setenvif status
a2enmod actions expires fastcgi proxy proxy_http rewrite ssl
==== Configure iptables in Ubuntu ====
iptables -F
iptables -t nat -A POSTROUTING -s 192.168.253.0/24 -o eth0 -j MASQUERADE
iptables -A INPUT -p tcp -i eth0 --dport 443 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 80 -j ACCEPT
iptables -A INPUT -p udp -i eth0 --dport 53 -j ACCEPT
iptables -A INPUT -p udp -i eth0 --sport 53 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 1723 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 47 -j ACCEPT
iptables -A INPUT -p gre -j ACCEPT
iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A INPUT -j REJECT
iptables-save > /etc/iptables.up.rules
#at the end of the primary network interface (eth0) config add:
pre-up iptables-restore < /etc/iptables.up.rules
==== Install Oracle JDK7 ====
wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F" "http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"
sudo mkdir -p /usr/lib/jvm
sudo tar -xvf jdk-7u51-linux-x64.tar.gz -C /usr/lib/jvm
sudo update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jvm/jdk1.7.0_51/bin/java" 1
sudo update-alternatives --install "/usr/bin/javac" "javac" "/usr/lib/jvm/jdk1.7.0_51/bin/javac" 1
sudo update-alternatives --install "/usr/bin/javaws" "javaws" "/usr/lib/jvm/jdk1.7.0_51/bin/javaws" 1
sudo update-alternatives --config java
sudo bash -c "cat >> /etc/profile" <
==== Install Microsoft Core Fonts ====
sudo apt-get install msttcorefonts
==== Install 32-bit libraries ====
sudo apt-get install ia32-libs (deprecated since Ubuntu 13.10)
sudo apt-get install xxx:i386 (after Ubuntu 13.10)
==== Install linux headers ====
sudo apt-get install build-essential linux-headers-$(uname -r)
==== Remove old kernel image ====
kernelver=$(uname -r | sed -r 's/-[a-z]+//')
dpkg -l linux-{image,headers,image-extra}-"[0-9]*" | awk '/ii/{print $2}' | grep -ve $kernelver | xargs dpkg --force-all -P
==== Install Lyx ====
sudo apt-add-repository ppa:texlive-backports/ppa
sudo apt-get update
sudo apt-get install texlive-base texlive-xetex texlive-lang-cjk cjk-latex latex-cjk-all lyx
==== Fix console output "mountall: disconnected from plymouth" ====
sudo -i
echo FRAMEBUFFER=y > /etc/initramfs-tools/conf.d/splash
update-initramfs -u
==== Backup and recover installed package via apt-get ====
sudo dpkg --get-selections > pkg.txt
sudo dpkg --set-selections < pkg.txt
sudo apt-get -u dselect-upgrade
==== Install Chrome stable version ====
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
sudo sh -c 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
sudo apt-get update
sudo apt-get install google-chrome-stable
==== Get DHCP server address ====
grep "dhcp-server-identifier" /var/lib/dhcp/dhclient.*.leases | awk '{print $3}'
==== Disable IPv6 ====
cat>> /etc/sysctl.conf <
==== Preferring IPv4 over IPv6 for apt-get ====
# See more detail information on http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses
sudo sed -i 's|#\(precedence \+::ffff:0:0/96 \+100\)|\1|' /etc/gai.conf
==== Using NTP to sync time ====
sudo apt-get install ntp
# modify ntp configuration
sudo nano /etc/ntp.conf
sudo service ntp restart
==== Configure proxy for apt ====
sudo bash -c "cat >> /etc/apt/apt.conf" <:/";
Acquire::https::proxy "https://:/";
Acquire::ftp::proxy "ftp://:/";
Acquire::socks::proxy "socks://:/";
EOF
==== Install NFS client ====
sudo apt-get install portmap nfs-common lsb-core
==== Change home folder name back to English ====
export LANG=en_US
xdg-user-dirs-gtk-update
// Relogin
==== Enable cron log ====
# Uncomment cron.* lines in /etc/rsyslog.d/50-default.conf
service rsyslog restart
service cron restart
more /var/log/cron.log
==== Set locale ====
1. vi /var/lib/locales/supported.d/en
zh_CN.UTF-8 UTF-8
2. locale-gen
3. vi /etc/environment
LANG="zh_CN.UTF-8"
LANGUAGE="zh_CN:zh:en_US:en"
4. reboot
==== Generate self-signed certificate using OpenSSL ====
cd /etc/ssl
mkdir demoCA
mkdir demoCA/newcerts
touch demoCA/index.txt
touch demoCA/serial
echo "01" > demoCA/serial
openssl genrsa -out server.key 1024
openssl req -new -key server.key -out server.csr -config openssl.cnf -days 3650
# Generate CA
openssl req -new -x509 -keyout ca.key -out ca.crt -config openssl.cnf -days 3650
# Sign cert using CA
openssl ca -in server.csr -out server.crt -cert ca.crt -keyfile ca.key -config openssl.cnf -days 3650
cat server.crt server.key > server.pem
===== RHEL =====
==== Change kernel load sequence in RHEL 6.x ====
# Add following parameters behind kernel parameter in /boot/grub/grub.conf
rdloaddriver=mptbase,mptspi,mptscsih,vmw_pvscsi rdblacklist=ata_piix