Linux Tricks
Linux Common
cat xxx.xml | xmllint --format -
Remount filesystem as read-write
iostat
meminfo/free
mpstat
netstat
nmon
pmap
ps/pstree
sar
strace
tcpdump
top
uptime
vmstat
wireshark
Use ipref to test network
#UDP mode
#Server side
iperf -u -s
#Client side
iperf -u -c 192.168.1.1 -b 100M -t 60
#Using 30 threads with 5Mbps each
iperf -u -c 192.168.1.1 -b 5M -P 30 -t 60
#Test both download and upload bandwidth
iperf -u -c 192.168.1.1 -b 100M -d -t 60
#TCP mode
#Server side
iperf -s
#Client side
iperf -c 192.168.1.1 -t 60
#Using 30 threads
iperf -c 192.168.1.1 -P 30 -t 60
#Test both download and upload bandwidth
iperf -c 192.168.1.1 -d -t 60
Add mount --bind info in /etc/fstab
/mnt/hdd1/test /home/test/ext none rw,bind 0 0
Scan local IPv4 opened ports
nc -zv localhost 1-65535 2>&1 | grep -v "failed"
netstat -lnt4 | awk '{print $4}' | cut -f2 -d: | grep -o '[0-9]*' | sort -n
Scan opened ports in given subnet
nmap -p 22 192.168.1.0/24
Get system serial number
sudo dmidecode --type 1 | grep 'Serial Number'
Hot resize ext2/3/4 partition
# Dangerous, following commands will force update partition table!
sudo -i
DISK=/dev/sdb
sfdisk -d $DISK > ./partition_table
total_sectors=`fdisk -l $DISK | grep 'total [0-9]\+ sectors' | sed 's|.*total \([0-9]\+\) sectors.*|\1|'`
last_partition=`fdisk -l $DISK | grep "^$DISK[0-9]\+" | awk '{print $1}'| tail -1`
start_sector=`cat ./partition_table | grep "$last_partition " | sed 's|.*start= *\([0-9]\+\).*|\1|'`
new_size=`expr $total_sectors - $start_sector`
sed "s|\($last_partition .*size= *\)[0-9]\+\(.*\)|\1$new_size\2|" ./partition_table > ./new_partition_table
sfdisk $DISK < ./new_partition_table
resize2fs $last_partition
wget --server-response --spider {URL}
curl -I {URL}
w3m -dump_head {URL}
Calculate MD5 hash for string
Sed usage
# Print line 10-20
sed -n '10,20p' {FILE}
Print zombie process PID
# Kill zombie process
ps -eal | awk '{ if ($2 == "Z") {print $4}}' | sudo kill -9
# Print zombie process's parent PID
ps -eal | awk '{ if ($2 == "Z") {print $5}}' | sort -u
Print process tree
Sort process by physically resident memory
ps -eo "pid,rss,vsz,size,cputime,%cpu,cmd" --width 120 --sort rss,vsz | sort -k2 -rn | head -10
Show files opened by given process
Split large file and merge back
split –b 1G /xxx_large_file /tmp/split_files
cat /tmp/split_files* > /tmp/merged
Execute a program every 5 seconds and fresh output
Update DDNS
nsupdate <<EOF
update add $HOST 86400 A $IP
send
EOF
Package ISO
mkisofs -o {OUTPUT_ISO} -Jrv {ISO_VOLUMN_NAME} {INPUT_FOLDER}
Package Linux bootable ISO
mkisofs -o "{OUTPUT_ISO}" -Jrvb "isolinux/isolinux.bin" -c "isolinux/boot.cat" -no-emul-boot -boot-load-size 4 -boot-info-table -allow-leading-dots -V "Linux_Boot" "{INPUT_FOLDER}"
Download webpage recursively
wget -r --no-parent -nv -nH --cut-dirs=1 --reject "index.html*" http://xxxx/yyy/zzz -P {LOCAL_DIR}
View current network connect and pid
Some useful bash alias
alias wget="wget --no-check-certificate"
alias grep="grep -i --color=auto -s"
alias ssh="ssh -o StrictHostKeyChecking=no"
alias scp="scp -o StrictHostKeyChecking=no"
Generate random string
openssl rand -hex 8 2>/dev/null
< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-16}; echo;
Add swap file
sudo dd if=/dev/zero of=/swapfile bs=1024 count=512k
sudo mkswap /swapfile
sudo swapon /swapfile
swapon -s
sudo bash -c "cat >> /etc/fstab" <<EOF
/swapfile none swap sw 0 0
EOF
Mount root in r/w mode
SSH auto login
# ssh-keygen -t rsa -b 2048
cat ~/.ssh/id_rsa.pub | ssh user@server "cat >> ~/.ssh/authorized_keys"
Copy a disk over SSH
ssh root@{HOST} "dd if=/dev/sdb bs=1k conv=sync,noerror | gzip -c" | gunzip -c | dd of=/dev/sdb bs=1M
Bypass ssh/scp host verification
ssh -o StrictHostKeyChecking=no user@host
scp -o StrictHostKeyChecking=no ...
or
cat >> ~/.ssh/config <<EOF
Host 192.168.0.*
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
EOF
Small tricks in linux shell
sudo !! #Use sudo to execute last command
python -m SimpleHTTPServer #A small web server in Python
:w !sudo tee % #Vim command to force write file with permissions
cd - #Cd to last directory
^foo^bar #Replace foo with bar in last command and then execute it
cp filename{,.bak} #Shortcut for backup file
mtr IP_address #Test network
!whatever:p #Print history commands contains whatever without execute it
cal #Show calendar
${variable:0:5} #Cut string variable
stat -c %U xxx #Get file owner
cmd #Skip bash history, note the heading space
printf '%100s\n' | tr ' ' - #Pring a line using '-'
vim + {FILE} #Open file and jump to bottom
Bypass sudo password
sudo bash -c "cat >> /etc/sudoers" <<EOF
{username} ALL=(ALL) NOPASSWD: ALL
EOF
Add a Linux user account without password
adduser --disabled-password --no-create-home --gecos "" {username}
Clear disk cache
sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches"
Turn off filesystem check
/sbin/tune2fs -c 0 -i 0 /dev/{device_name}
Test harddisk speed
# Test cache speed
sudo hdparm -Tt /dev/sda
# Test continuous large file write speed
time dd if=/dev/zero of=/tmp/2G_tmp bs=1M count=2kB conv=fdatasync
# Test continuous large file read speed
time dd if=/tmp/2G_tmp bs=1M count=2kB | dd of=/dev/null 2>/dev/null
rm -rf /tmp/2G_tmp
# Test sequential/random read/write speed from 4k to 16k
iozone -Rab iozone-result_speed.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k
# Test sequential/random read/write IOPS from 4k to 16k
iozone -Rab iozone-result_iops.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k -O
# Use bonnie++ to test disk performance
sudo bonnie++ -u root
Get current disk IO load every 5 seconds
Scan hot-added SCSI disk
echo "- - -" > /sys/class/scsi_host/host{scsi_host_id}/scan
Refresh SCSI disk size
echo 1 > /sys/block/{device_name}/device/rescan
Convert Linux text file to Windows style
Remove ^M in text file
cat xxx.txt | tr -d "^M" > xxx_new.txt
sed -i "s/^M//g" xxx.txt
Remove dead symbolic links
find -L -type l -delete
#Avoid follow the link recursively
find -L -maxdepth 1 -type l -delete
Delete empty directory recursively
find ./ -depth -empty -type d -exec rmdir -v {} \;
Find top 10 biggest files in current directory recursively
find . -printf '%s %p\n'| sort -nr | head -10
sudo lshw
sudo dmidecode -t memory
List all PCI devices on bus tree
List all USB devices on bus tree
List all block device
Test whether can establish TCP connection
timeout 1 bash -c 'echo > /dev/tcp/10.110.160.222/22 && echo "accessible"' || echo "not accessible"
nc -z -w 1 10.110.160.2 22 && echo "accessible" || echo "not accessible"
#sudo apt-get install lvm2
sudo pvcreate /dev/sda6 /dev/sda7
#Check physical volumes
sudo pvscan
sudo pvdisplay
sudo vgcreate vol_grp1 /dev/sda6 /dev/sda7
sudo vgdisplay
#sudo lvcreate -L 20G vg1
# Use all the free space to create logical volume
sudo lvcreate –l 100%FREE –n logical_vol1 vol_grp1
#Extend logical volume to 100GB
sudo lvextend -L100G /dev/vol_grp1/logical_vol1
lsb_release -a
cat /etc/*release
Dump HTTP POST/GET request using tcpdump
# 0x504f5354 is ASCII code for POST, change 0xdeadbeaf to match url accordingly.
tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x504f5354) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 5):4] = 0xdeadbeaf)' -w ~/post.log -U -A
# 0x47455420 is ASCII code for GET, change 0xdeadbeaf to match url accordingly.
tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 4):4] = 0xdeadbeaf)' -w ~/get.log -U -A
Notes about tcpdump filters
Notes about tcpdump filters
# Source: http://staff.washington.edu/dittrich/talks/core02/tools/tcpdump-filters.txt
NOTES ABOUT TCPDUMP FILTERS
===========================
Expression Meaning
========== =======
[x:y] start at offset x from the beginning of packet and read y bytes
[x] abbreviation for [x:1]
proto[x:y] start at offset x into the proto header and read y bytes
p[x:y] & z = 0 p[x:y] has none of the bits selected by z
p[x:y] & z != 0 p[x:y] has any of the bits selected by z
p[x:y] & z = z p[x:y] has all of the bits selected by z
p[x:y] = z p[x:y] has only the bits selected by z
the usual rules about operator precedence apply; nesting things inside brackets
is probably a good plan. you'll probably want to put the filter into a file or
at least single-quote it on the commandline to stop the shell from interpreting
the metacharacters. !([:])&
Interesting Parts of a Packet
=============================
ip[0] & 0xf0 high nibble: IP version. almost always 4
ip[0] & 0x0f low nibble: header length in 4octet words. should be 5
ip[1] type of service/QoS/DiffServ
ip[2:2] total length of datagram in octets
ip[4:2] IP ID number
ip[6] & 0x80 reserved bit (possibly used for ECN)
ip[6] & 0x40 DF bit
ip[6] & 0x20 MF bit
ip[6:2] & 0x1fff fragment offset (number of 8octet blocks)
ip[8] ttl
ip[9] protocol
ip[10:2] header checksum
ip[12:4] source IP
ip[16:4] destination IP
ip[20..60] there better not be any options in here...
Interesting Parts of an ICMP Message
====================================
icmp[0] type
icmp[1] code
icmp[2:2] checksum
icmp[4...] payload
Interesting Parts of a UDP header
=================================
udp[0:2] source port
udp[2:2] destination port
udp[4:2] datagram length
udp[6:2] UDP checksum
Interesting Parts of a TCP header
=================================
tcp[0:2] source port
tcp[2:2] destination port
tcp[4:4] sequence number
tcp[8:4] acknowledgement number
tcp[12] header length
tcp[13] tcp flags
tcp[14:2] window size
tcp[16:2] checksum
tcp[18:2] urgent pointer
tcp[20..60] options or data
Flags Numerically Meaning
===== =========== =======
---- --S- 0000 0010 = 0x02 normal syn
---A --S- 0001 0010 = 0x12 normal syn-ack
---A ---- 0001 0000 = 0x10 normal ack
--UA P--- 0011 1000 = 0x38 psh-urg-ack. interactive stuff like ssh
---A -R-- 0001 0100 = 0x14 rst-ack. it happens.
---- --SF 0000 0011 = 0x03 syn-fin scan
--U- P--F 0010 1001 = 0x29 urg-psh-fin. nmap fingerprint packet
-Y-- ---- 0100 0000 = 0x40 anything >= 0x40 has a reserved bit set
XY-- ---- 1100 0000 = 0xC0 both reserved bits set
XYUA PRSF 1111 1111 = 0xFF FULL_XMAS scan
ICMP Types and Codes
====================
0 ECHOREPLY
3 UNREACHABLE
3:0 NET
3:1 HOST
3:2 PROTOCOL
3:3 PORT
3:4 NEEDFRAG
3:5 SRC_ROUTE_FAILED
3:6 NET_UNKNOWN
3:7 HOST_UNKNOWN
3:8 SRC_HOST_ISOLATED
3:9 NET_PROHIB
3:10 HOST_PROHIB
3:11 BAD_TOS_FOR_NET
3:12 BAD_TOS_FOR_HOST
3:13 FILTER_PROHIB
3:14 HOST_PRECEDENCE_VIOLATION
3:15 PRECEDENCE_CUTOFF
4 SOURCEQUENCH
5 REDIRECT
5:0 NET
5:1 HOST
5:2 TOSNET
5:3 TOSHOST
8 ECHO
9 ROUTERADVERT
10 ROUTERSOLICIT
11 TIME_EXCEEDED
11:0 IN_TRANSIT
11:1 DURING_FRAG_REASSEMBLY
12 PARAMETER_PROBLEM
12:1 MISSING_OPT_FOR_REQUEST
13 TSTAMP_REQ
14 TSTAMP_REPLY
15 INFO_REQ
16 INFO_REPLY
17 NETMASK_REQ
18 NETMASK_REPLY
Examples
--------
is SYN. nothing else.
tcp[13] = 0x02
contains SYN. we don't care what else...
(tcp[13] & 0x02) != 0
is some kind of SYN-FIN. Bad news
(tcp[13] & 0x03) = 3
land attack
ip[12:4] = ip[16:4]
winnuke
(tcp[2:2] = 139) && (tcp[13] & 0x20 != 0) && (tcp[19] & 0x01 = 1)
things other than ACK/PSH
(tcp[13] & 0xe7) != 0
initial fragments
(ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff = 0)
intervening fragments
(ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff != 0)
terminal fragments
(ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0)
has ip options (or is truncated, or is just some sort of freak...)
(ip[0] & 0x0f) != 5
ping o' death and its ilk (any oversized IP-transported data...)
((ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0)) && \
((65535 < (ip[2:2] + 8*(ip[6:2] & 0x1fff))
Ubuntu/Debian
Automatically install dependencies with "dpkg -i"
dpkg -i
If it needs dependencies, get them automatically with apt-get -y -f –force-yes install. dpkg -i has generated the list of unresolved dependencies, apt-get -f will just pick it up.
Run the dpkg -i again
Ubuntu/Debian auto login
sudo apt-get install mingetty
vim nano /etc/inittab
#change the line 1:2345:respawn:/sbin/getty 38400 tty1 to 1:2345:respawn:/sbin/mingetty --autologin <user-name> --noclear tty1
Connect PPTP VPN
pptpsetup --create hackerzhou.me --server {SERVER_IP} --username {USERNAME} --password {PASSWORD} --encrypt --start
Install TrueType fonts
sudo apt-get install ttf-mscorefonts-installer
Install KDE
Check rootkit and security holes
sudo apt-get install rkhunter
rkhunter --checkall
Setup Samba with management UI on Ubuntu
sudo apt-get install samba samba-common
sudo apt-get install system-config-samba
sudo smbpasswd -a xxxxx
Setup MySQL for Python on Ubuntu
sudo apt-get install python-pip python-dev libmysqlclient-dev
pip install -U pip
pip install MySQL-python
Setup VLAN on Ubuntu
# download latest vlan deb from http://www.ubuntuupdates.org/vlan
sudo dpkg -i vlan_1.9-3ubuntu10_amd64.deb
sudo su -c 'echo "8021q" >> /etc/modules'
sudo bash -c "cat >> /etc/network/interfaces" <<EOF
auto eth1.{vlan_id}
iface eth1.{vlan_id} inet static
address xx.xx.xx.xx
netmask xx.xx.xx.xx
gateway xx.xx.xx.xx
vlan-raw-device eth1
dns-nameservers xx.xx.xx.xx
EOF
Install & config pptpd
sudo apt-get install pptpd
sudo bash -c "cat>> /etc/pptpd.conf" <<EOF
localip 192.168.253.1
remoteip 192.168.253.2-238,192.168.253.245
EOF
sudo bash -c "cat>> /etc/ppp/pptpd-options" <<EOF
ms-dns 8.8.8.8
ms-dns 8.8.4.4
EOF
sudo bash -c "cat>> /etc/ppp/chap-secrets " <<EOF
username * "password" *
EOF
Install Node.js
sudo apt-get install python-software-properties python g++ make
sudo add-apt-repository ppa:chris-lea/node.js
sudo apt-get update
sudo apt-get install nodejs
Set timezone to GMT+8
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
or
sudo dpkg-reconfigure tzdata
Quick setup VPS
apt-get install axel p7zip-full unzip mysql-server apache2 libapache2-mod-fastcgi php5 php5-fpm php5-cgi php5-mysql phpmyadmin
a2dismod autoindex cgi negotiation php5 reqtimeout setenvif status
a2enmod actions expires fastcgi proxy proxy_http rewrite ssl
iptables -F
iptables -t nat -A POSTROUTING -s 192.168.253.0/24 -o eth0 -j MASQUERADE
iptables -A INPUT -p tcp -i eth0 --dport 443 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 80 -j ACCEPT
iptables -A INPUT -p udp -i eth0 --dport 53 -j ACCEPT
iptables -A INPUT -p udp -i eth0 --sport 53 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 1723 -j ACCEPT
iptables -A INPUT -p tcp -i eth0 --dport 47 -j ACCEPT
iptables -A INPUT -p gre -j ACCEPT
iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A INPUT -j REJECT
iptables-save > /etc/iptables.up.rules
#at the end of the primary network interface (eth0) config add:
pre-up iptables-restore < /etc/iptables.up.rules
Install Oracle JDK7
wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F" "http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"
sudo mkdir -p /usr/lib/jvm
sudo tar -xvf jdk-7u51-linux-x64.tar.gz -C /usr/lib/jvm
sudo update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jvm/jdk1.7.0_51/bin/java" 1
sudo update-alternatives --install "/usr/bin/javac" "javac" "/usr/lib/jvm/jdk1.7.0_51/bin/javac" 1
sudo update-alternatives --install "/usr/bin/javaws" "javaws" "/usr/lib/jvm/jdk1.7.0_51/bin/javaws" 1
sudo update-alternatives --config java
sudo bash -c "cat >> /etc/profile" <<EOF
export JAVA_HOME=/usr/lib/jvm/jdk1.7.0_51
EOF
java -version
Install Microsoft Core Fonts
sudo apt-get install msttcorefonts
Install 32-bit libraries
sudo apt-get install ia32-libs (deprecated since Ubuntu 13.10)
sudo apt-get install xxx:i386 (after Ubuntu 13.10)
sudo apt-get install build-essential linux-headers-$(uname -r)
Remove old kernel image
kernelver=$(uname -r | sed -r 's/-[a-z]+//')
dpkg -l linux-{image,headers,image-extra}-"[0-9]*" | awk '/ii/{print $2}' | grep -ve $kernelver | xargs dpkg --force-all -P
Install Lyx
sudo apt-add-repository ppa:texlive-backports/ppa
sudo apt-get update
sudo apt-get install texlive-base texlive-xetex texlive-lang-cjk cjk-latex latex-cjk-all lyx
Fix console output "mountall: disconnected from plymouth"
sudo -i
echo FRAMEBUFFER=y > /etc/initramfs-tools/conf.d/splash
update-initramfs -u
Backup and recover installed package via apt-get
sudo dpkg --get-selections > pkg.txt
sudo dpkg --set-selections < pkg.txt
sudo apt-get -u dselect-upgrade
Install Chrome stable version
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
sudo sh -c 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
sudo apt-get update
sudo apt-get install google-chrome-stable
Get DHCP server address
grep "dhcp-server-identifier" /var/lib/dhcp/dhclient.*.leases | awk '{print $3}'
Disable IPv6
cat>> /etc/sysctl.conf <<EOF
# Disable IPv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
EOF
Preferring IPv4 over IPv6 for apt-get
# See more detail information on http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses
sudo sed -i 's|#\(precedence \+::ffff:0:0/96 \+100\)|\1|' /etc/gai.conf
Using NTP to sync time
sudo apt-get install ntp
# modify ntp configuration
sudo nano /etc/ntp.conf
sudo service ntp restart
sudo bash -c "cat >> /etc/apt/apt.conf" <<EOF
Acquire::http::proxy "http://<proxy_host>:<proxy_port>/";
Acquire::https::proxy "https://<proxy_host>:<proxy_port>/";
Acquire::ftp::proxy "ftp://<proxy_host>:<proxy_port>/";
Acquire::socks::proxy "socks://<proxy_host>:<proxy_port>/";
EOF
Install NFS client
sudo apt-get install portmap nfs-common lsb-core
Change home folder name back to English
export LANG=en_US
xdg-user-dirs-gtk-update
// Relogin
Enable cron log
# Uncomment cron.* lines in /etc/rsyslog.d/50-default.conf
service rsyslog restart
service cron restart
more /var/log/cron.log
Set locale
1. vi /var/lib/locales/supported.d/en
zh_CN.UTF-8 UTF-8
2. locale-gen
3. vi /etc/environment
LANG="zh_CN.UTF-8"
LANGUAGE="zh_CN:zh:en_US:en"
4. reboot
Generate self-signed certificate using OpenSSL
cd /etc/ssl
mkdir demoCA
mkdir demoCA/newcerts
touch demoCA/index.txt
touch demoCA/serial
echo "01" > demoCA/serial
openssl genrsa -out server.key 1024
openssl req -new -key server.key -out server.csr -config openssl.cnf -days 3650
# Generate CA
openssl req -new -x509 -keyout ca.key -out ca.crt -config openssl.cnf -days 3650
# Sign cert using CA
openssl ca -in server.csr -out server.crt -cert ca.crt -keyfile ca.key -config openssl.cnf -days 3650
cat server.crt server.key > server.pem
RHEL
Change kernel load sequence in RHEL 6.x
# Add following parameters behind kernel parameter in /boot/grub/grub.conf
rdloaddriver=mptbase,mptspi,mptscsih,vmw_pvscsi rdblacklist=ata_piix