====== Linux Tricks ====== ===== Linux Common ===== ==== XML beautiful format command ==== cat xxx.xml | xmllint --format - ==== Remount filesystem as read-write ==== mount -o rw,remount / ==== Useful tools to monitor linux ==== iostat meminfo/free mpstat netstat nmon pmap ps/pstree sar strace tcpdump top uptime vmstat wireshark ==== Use ipref to test network ==== #UDP mode #Server side iperf -u -s #Client side iperf -u -c 192.168.1.1 -b 100M -t 60 #Using 30 threads with 5Mbps each iperf -u -c 192.168.1.1 -b 5M -P 30 -t 60 #Test both download and upload bandwidth iperf -u -c 192.168.1.1 -b 100M -d -t 60 #TCP mode #Server side iperf -s #Client side iperf -c 192.168.1.1 -t 60 #Using 30 threads iperf -c 192.168.1.1 -P 30 -t 60 #Test both download and upload bandwidth iperf -c 192.168.1.1 -d -t 60 ==== Add mount --bind info in /etc/fstab ==== /mnt/hdd1/test /home/test/ext none rw,bind 0 0 ==== Scan local IPv4 opened ports ==== nc -zv localhost 1-65535 2>&1 | grep -v "failed" netstat -lnt4 | awk '{print $4}' | cut -f2 -d: | grep -o '[0-9]*' | sort -n ==== Scan opened ports in given subnet ==== nmap -p 22 192.168.1.0/24 ==== Get system serial number ==== sudo dmidecode --type 1 | grep 'Serial Number' ==== Hot resize ext2/3/4 partition ==== # Dangerous, following commands will force update partition table! sudo -i DISK=/dev/sdb sfdisk -d $DISK > ./partition_table total_sectors=`fdisk -l $DISK | grep 'total [0-9]\+ sectors' | sed 's|.*total \([0-9]\+\) sectors.*|\1|'` last_partition=`fdisk -l $DISK | grep "^$DISK[0-9]\+" | awk '{print $1}'| tail -1` start_sector=`cat ./partition_table | grep "$last_partition " | sed 's|.*start= *\([0-9]\+\).*|\1|'` new_size=`expr $total_sectors - $start_sector` sed "s|\($last_partition .*size= *\)[0-9]\+\(.*\)|\1$new_size\2|" ./partition_table > ./new_partition_table sfdisk $DISK < ./new_partition_table resize2fs $last_partition ==== Print HTTP headers ==== wget --server-response --spider {URL} curl -I {URL} w3m -dump_head {URL} ==== Calculate MD5 hash for string ==== echo -n 'hello' | md5sum ==== Validate XML format ==== xmllint --noout test.xml ==== Sed usage ==== # Print line 10-20 sed -n '10,20p' {FILE} ==== Print zombie process PID ==== # Kill zombie process ps -eal | awk '{ if ($2 == "Z") {print $4}}' | sudo kill -9 # Print zombie process's parent PID ps -eal | awk '{ if ($2 == "Z") {print $5}}' | sort -u ==== Print process tree ==== ps axwef pstree ==== Sort process by physically resident memory ==== ps -eo "pid,rss,vsz,size,cputime,%cpu,cmd" --width 120 --sort rss,vsz | sort -k2 -rn | head -10 ==== Show files opened by given process ==== lsof -p {PID} ==== Split large file and merge back ==== split –b 1G /xxx_large_file /tmp/split_files cat /tmp/split_files* > /tmp/merged ==== Execute a program every 5 seconds and fresh output ==== watch -n 5 {COMMAND} ==== Update DDNS ==== nsupdate < ==== Package ISO ==== mkisofs -o {OUTPUT_ISO} -Jrv {ISO_VOLUMN_NAME} {INPUT_FOLDER} ==== Package Linux bootable ISO ==== mkisofs -o "{OUTPUT_ISO}" -Jrvb "isolinux/isolinux.bin" -c "isolinux/boot.cat" -no-emul-boot -boot-load-size 4 -boot-info-table -allow-leading-dots -V "Linux_Boot" "{INPUT_FOLDER}" ==== Download webpage recursively ==== wget -r --no-parent -nv -nH --cut-dirs=1 --reject "index.html*" http://xxxx/yyy/zzz -P {LOCAL_DIR} ==== View current network connect and pid ==== sudo netstat -atnp ==== Some useful bash alias ==== alias wget="wget --no-check-certificate" alias grep="grep -i --color=auto -s" alias ssh="ssh -o StrictHostKeyChecking=no" alias scp="scp -o StrictHostKeyChecking=no" ==== Generate random string ==== openssl rand -hex 8 2>/dev/null < /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-16}; echo; ==== Add swap file ==== sudo dd if=/dev/zero of=/swapfile bs=1024 count=512k sudo mkswap /swapfile sudo swapon /swapfile swapon -s sudo bash -c "cat >> /etc/fstab" < ==== Mount root in r/w mode ==== mount -o remount,rw / ==== SSH auto login ==== # ssh-keygen -t rsa -b 2048 cat ~/.ssh/id_rsa.pub | ssh user@server "cat >> ~/.ssh/authorized_keys" ==== Copy a disk over SSH ==== ssh root@{HOST} "dd if=/dev/sdb bs=1k conv=sync,noerror | gzip -c" | gunzip -c | dd of=/dev/sdb bs=1M ==== Bypass ssh/scp host verification ==== ssh -o StrictHostKeyChecking=no user@host scp -o StrictHostKeyChecking=no ... or cat >> ~/.ssh/config < ==== Small tricks in linux shell ==== sudo !! #Use sudo to execute last command python -m SimpleHTTPServer #A small web server in Python :w !sudo tee % #Vim command to force write file with permissions cd - #Cd to last directory ^foo^bar #Replace foo with bar in last command and then execute it cp filename{,.bak} #Shortcut for backup file mtr IP_address #Test network !whatever:p #Print history commands contains whatever without execute it cal #Show calendar ${variable:0:5} #Cut string variable stat -c %U xxx #Get file owner cmd #Skip bash history, note the heading space printf '%100s\n' | tr ' ' - #Pring a line using '-' vim + {FILE} #Open file and jump to bottom ==== Bypass sudo password ==== sudo bash -c "cat >> /etc/sudoers" < ==== Add a Linux user account without password ==== adduser --disabled-password --no-create-home --gecos "" {username} ==== Clear disk cache ==== sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" ==== Turn off filesystem check ==== /sbin/tune2fs -c 0 -i 0 /dev/{device_name} ==== Test harddisk speed ==== # Test cache speed sudo hdparm -Tt /dev/sda # Test continuous large file write speed time dd if=/dev/zero of=/tmp/2G_tmp bs=1M count=2kB conv=fdatasync # Test continuous large file read speed time dd if=/tmp/2G_tmp bs=1M count=2kB | dd of=/dev/null 2>/dev/null rm -rf /tmp/2G_tmp # Test sequential/random read/write speed from 4k to 16k iozone -Rab iozone-result_speed.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k # Test sequential/random read/write IOPS from 4k to 16k iozone -Rab iozone-result_iops.xls -s 16G -i 0 -i 1 -i 2 -y 4k -q 16k -O # Use bonnie++ to test disk performance sudo bonnie++ -u root ==== Get current disk IO load every 5 seconds ==== sudo iostat -x 5 ==== Scan hot-added SCSI disk ==== echo "- - -" > /sys/class/scsi_host/host{scsi_host_id}/scan ==== Refresh SCSI disk size ==== echo 1 > /sys/block/{device_name}/device/rescan ==== Convert Linux text file to Windows style ==== sed -i 's/$/\r/' xxx.txt ==== Remove ^M in text file ==== cat xxx.txt | tr -d "^M" > xxx_new.txt sed -i "s/^M//g" xxx.txt ==== Grep a file to remove blank and comment (#) lines ==== grep -v "^#\|^$" ==== Remove dead symbolic links ==== find -L -type l -delete #Avoid follow the link recursively find -L -maxdepth 1 -type l -delete ==== Delete empty directory recursively ==== find ./ -depth -empty -type d -exec rmdir -v {} \; ==== Find top 10 biggest files in current directory recursively ==== find . -printf '%s %p\n'| sort -nr | head -10 ==== Show physical memory slot information ==== sudo lshw sudo dmidecode -t memory ==== List all PCI devices on bus tree ==== lspci -tv ==== List all USB devices on bus tree ==== lsusb -tv ==== List all block device==== lsblk ==== Test whether can establish TCP connection ==== timeout 1 bash -c 'echo > /dev/tcp/10.110.160.222/22 && echo "accessible"' || echo "not accessible" nc -z -w 1 10.110.160.2 22 && echo "accessible" || echo "not accessible" ==== Configure LVM partition ==== #sudo apt-get install lvm2 sudo pvcreate /dev/sda6 /dev/sda7 #Check physical volumes sudo pvscan sudo pvdisplay sudo vgcreate vol_grp1 /dev/sda6 /dev/sda7 sudo vgdisplay #sudo lvcreate -L 20G vg1 # Use all the free space to create logical volume sudo lvcreate –l 100%FREE –n logical_vol1 vol_grp1 #Extend logical volume to 100GB sudo lvextend -L100G /dev/vol_grp1/logical_vol1 ==== Lookup distro information ==== lsb_release -a cat /etc/*release ==== Dump HTTP POST/GET request using tcpdump ==== # 0x504f5354 is ASCII code for POST, change 0xdeadbeaf to match url accordingly. tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x504f5354) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 5):4] = 0xdeadbeaf)' -w ~/post.log -U -A # 0x47455420 is ASCII code for GET, change 0xdeadbeaf to match url accordingly. tcpdump 'tcp dst port 80 and (tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420) and (tcp[(((tcp[12:1] & 0xf0) >> 2) + 4):4] = 0xdeadbeaf)' -w ~/get.log -U -A # Source: http://staff.washington.edu/dittrich/talks/core02/tools/tcpdump-filters.txt NOTES ABOUT TCPDUMP FILTERS =========================== Expression Meaning ========== ======= [x:y] start at offset x from the beginning of packet and read y bytes [x] abbreviation for [x:1] proto[x:y] start at offset x into the proto header and read y bytes p[x:y] & z = 0 p[x:y] has none of the bits selected by z p[x:y] & z != 0 p[x:y] has any of the bits selected by z p[x:y] & z = z p[x:y] has all of the bits selected by z p[x:y] = z p[x:y] has only the bits selected by z the usual rules about operator precedence apply; nesting things inside brackets is probably a good plan. you'll probably want to put the filter into a file or at least single-quote it on the commandline to stop the shell from interpreting the metacharacters. !([:])& Interesting Parts of a Packet ============================= ip[0] & 0xf0 high nibble: IP version. almost always 4 ip[0] & 0x0f low nibble: header length in 4octet words. should be 5 ip[1] type of service/QoS/DiffServ ip[2:2] total length of datagram in octets ip[4:2] IP ID number ip[6] & 0x80 reserved bit (possibly used for ECN) ip[6] & 0x40 DF bit ip[6] & 0x20 MF bit ip[6:2] & 0x1fff fragment offset (number of 8octet blocks) ip[8] ttl ip[9] protocol ip[10:2] header checksum ip[12:4] source IP ip[16:4] destination IP ip[20..60] there better not be any options in here... Interesting Parts of an ICMP Message ==================================== icmp[0] type icmp[1] code icmp[2:2] checksum icmp[4...] payload Interesting Parts of a UDP header ================================= udp[0:2] source port udp[2:2] destination port udp[4:2] datagram length udp[6:2] UDP checksum Interesting Parts of a TCP header ================================= tcp[0:2] source port tcp[2:2] destination port tcp[4:4] sequence number tcp[8:4] acknowledgement number tcp[12] header length tcp[13] tcp flags tcp[14:2] window size tcp[16:2] checksum tcp[18:2] urgent pointer tcp[20..60] options or data Flags Numerically Meaning ===== =========== ======= ---- --S- 0000 0010 = 0x02 normal syn ---A --S- 0001 0010 = 0x12 normal syn-ack ---A ---- 0001 0000 = 0x10 normal ack --UA P--- 0011 1000 = 0x38 psh-urg-ack. interactive stuff like ssh ---A -R-- 0001 0100 = 0x14 rst-ack. it happens. ---- --SF 0000 0011 = 0x03 syn-fin scan --U- P--F 0010 1001 = 0x29 urg-psh-fin. nmap fingerprint packet -Y-- ---- 0100 0000 = 0x40 anything >= 0x40 has a reserved bit set XY-- ---- 1100 0000 = 0xC0 both reserved bits set XYUA PRSF 1111 1111 = 0xFF FULL_XMAS scan ICMP Types and Codes ==================== 0 ECHOREPLY 3 UNREACHABLE 3:0 NET 3:1 HOST 3:2 PROTOCOL 3:3 PORT 3:4 NEEDFRAG 3:5 SRC_ROUTE_FAILED 3:6 NET_UNKNOWN 3:7 HOST_UNKNOWN 3:8 SRC_HOST_ISOLATED 3:9 NET_PROHIB 3:10 HOST_PROHIB 3:11 BAD_TOS_FOR_NET 3:12 BAD_TOS_FOR_HOST 3:13 FILTER_PROHIB 3:14 HOST_PRECEDENCE_VIOLATION 3:15 PRECEDENCE_CUTOFF 4 SOURCEQUENCH 5 REDIRECT 5:0 NET 5:1 HOST 5:2 TOSNET 5:3 TOSHOST 8 ECHO 9 ROUTERADVERT 10 ROUTERSOLICIT 11 TIME_EXCEEDED 11:0 IN_TRANSIT 11:1 DURING_FRAG_REASSEMBLY 12 PARAMETER_PROBLEM 12:1 MISSING_OPT_FOR_REQUEST 13 TSTAMP_REQ 14 TSTAMP_REPLY 15 INFO_REQ 16 INFO_REPLY 17 NETMASK_REQ 18 NETMASK_REPLY Examples -------- is SYN. nothing else. tcp[13] = 0x02 contains SYN. we don't care what else... (tcp[13] & 0x02) != 0 is some kind of SYN-FIN. Bad news (tcp[13] & 0x03) = 3 land attack ip[12:4] = ip[16:4] winnuke (tcp[2:2] = 139) && (tcp[13] & 0x20 != 0) && (tcp[19] & 0x01 = 1) things other than ACK/PSH (tcp[13] & 0xe7) != 0 initial fragments (ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff = 0) intervening fragments (ip[6] & 0x20 != 0) && (ip[6:2] & 0x1fff != 0) terminal fragments (ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0) has ip options (or is truncated, or is just some sort of freak...) (ip[0] & 0x0f) != 5 ping o' death and its ilk (any oversized IP-transported data...) ((ip[6] & 0x20 = 0) && (ip[6:2] & 0x1fff != 0)) && \ ((65535 < (ip[2:2] + 8*(ip[6:2] & 0x1fff)) ===== Ubuntu/Debian ===== ==== Automatically install dependencies with "dpkg -i" ==== - dpkg -i - If it needs dependencies, get them automatically with apt-get -y -f --force-yes install. dpkg -i has generated the list of unresolved dependencies, apt-get -f will just pick it up. - Run the dpkg -i again ==== Ubuntu/Debian auto login ==== sudo apt-get install mingetty vim nano /etc/inittab #change the line 1:2345:respawn:/sbin/getty 38400 tty1 to 1:2345:respawn:/sbin/mingetty --autologin --noclear tty1 ==== Connect PPTP VPN ==== pptpsetup --create hackerzhou.me --server {SERVER_IP} --username {USERNAME} --password {PASSWORD} --encrypt --start ==== Install TrueType fonts ==== sudo apt-get install ttf-mscorefonts-installer ==== Install KDE ==== [[http://wiki.ubuntu.org.cn/KDE]] ==== Check rootkit and security holes ==== sudo apt-get install rkhunter rkhunter --checkall ==== Setup Samba with management UI on Ubuntu ==== sudo apt-get install samba samba-common sudo apt-get install system-config-samba sudo smbpasswd -a xxxxx ==== Setup MySQL for Python on Ubuntu ==== sudo apt-get install python-pip python-dev libmysqlclient-dev pip install -U pip pip install MySQL-python ==== Setup VLAN on Ubuntu ==== # download latest vlan deb from http://www.ubuntuupdates.org/vlan sudo dpkg -i vlan_1.9-3ubuntu10_amd64.deb sudo su -c 'echo "8021q" >> /etc/modules' sudo bash -c "cat >> /etc/network/interfaces" < ==== Install & config pptpd ==== sudo apt-get install pptpd sudo bash -c "cat>> /etc/pptpd.conf" <> /etc/ppp/pptpd-options" <> /etc/ppp/chap-secrets " < ==== Install Node.js ==== sudo apt-get install python-software-properties python g++ make sudo add-apt-repository ppa:chris-lea/node.js sudo apt-get update sudo apt-get install nodejs ==== Set timezone to GMT+8 ==== ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime or sudo dpkg-reconfigure tzdata ==== Quick setup VPS ==== apt-get install axel p7zip-full unzip mysql-server apache2 libapache2-mod-fastcgi php5 php5-fpm php5-cgi php5-mysql phpmyadmin a2dismod autoindex cgi negotiation php5 reqtimeout setenvif status a2enmod actions expires fastcgi proxy proxy_http rewrite ssl ==== Configure iptables in Ubuntu ==== iptables -F iptables -t nat -A POSTROUTING -s 192.168.253.0/24 -o eth0 -j MASQUERADE iptables -A INPUT -p tcp -i eth0 --dport 443 -j ACCEPT iptables -A INPUT -p tcp -i eth0 --dport 80 -j ACCEPT iptables -A INPUT -p udp -i eth0 --dport 53 -j ACCEPT iptables -A INPUT -p udp -i eth0 --sport 53 -j ACCEPT iptables -A INPUT -p tcp -i eth0 --dport 1723 -j ACCEPT iptables -A INPUT -p tcp -i eth0 --dport 47 -j ACCEPT iptables -A INPUT -p gre -j ACCEPT iptables -A INPUT -i lo -j ACCEPT iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT iptables -A INPUT -j REJECT iptables-save > /etc/iptables.up.rules #at the end of the primary network interface (eth0) config add: pre-up iptables-restore < /etc/iptables.up.rules ==== Install Oracle JDK7 ==== wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F" "http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz" sudo mkdir -p /usr/lib/jvm sudo tar -xvf jdk-7u51-linux-x64.tar.gz -C /usr/lib/jvm sudo update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jvm/jdk1.7.0_51/bin/java" 1 sudo update-alternatives --install "/usr/bin/javac" "javac" "/usr/lib/jvm/jdk1.7.0_51/bin/javac" 1 sudo update-alternatives --install "/usr/bin/javaws" "javaws" "/usr/lib/jvm/jdk1.7.0_51/bin/javaws" 1 sudo update-alternatives --config java sudo bash -c "cat >> /etc/profile" < ==== Install Microsoft Core Fonts ==== sudo apt-get install msttcorefonts ==== Install 32-bit libraries ==== sudo apt-get install ia32-libs (deprecated since Ubuntu 13.10) sudo apt-get install xxx:i386 (after Ubuntu 13.10) ==== Install linux headers ==== sudo apt-get install build-essential linux-headers-$(uname -r) ==== Remove old kernel image ==== kernelver=$(uname -r | sed -r 's/-[a-z]+//') dpkg -l linux-{image,headers,image-extra}-"[0-9]*" | awk '/ii/{print $2}' | grep -ve $kernelver | xargs dpkg --force-all -P ==== Install Lyx ==== sudo apt-add-repository ppa:texlive-backports/ppa sudo apt-get update sudo apt-get install texlive-base texlive-xetex texlive-lang-cjk cjk-latex latex-cjk-all lyx ==== Fix console output "mountall: disconnected from plymouth" ==== sudo -i echo FRAMEBUFFER=y > /etc/initramfs-tools/conf.d/splash update-initramfs -u ==== Backup and recover installed package via apt-get ==== sudo dpkg --get-selections > pkg.txt sudo dpkg --set-selections < pkg.txt sudo apt-get -u dselect-upgrade ==== Install Chrome stable version ==== wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - sudo sh -c 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' sudo apt-get update sudo apt-get install google-chrome-stable ==== Get DHCP server address ==== grep "dhcp-server-identifier" /var/lib/dhcp/dhclient.*.leases | awk '{print $3}' ==== Disable IPv6 ==== cat>> /etc/sysctl.conf < ==== Preferring IPv4 over IPv6 for apt-get ==== # See more detail information on http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses sudo sed -i 's|#\(precedence \+::ffff:0:0/96 \+100\)|\1|' /etc/gai.conf ==== Using NTP to sync time ==== sudo apt-get install ntp # modify ntp configuration sudo nano /etc/ntp.conf sudo service ntp restart ==== Configure proxy for apt ==== sudo bash -c "cat >> /etc/apt/apt.conf" <:/"; Acquire::https::proxy "https://:/"; Acquire::ftp::proxy "ftp://:/"; Acquire::socks::proxy "socks://:/"; EOF ==== Install NFS client ==== sudo apt-get install portmap nfs-common lsb-core ==== Change home folder name back to English ==== export LANG=en_US xdg-user-dirs-gtk-update // Relogin ==== Enable cron log ==== # Uncomment cron.* lines in /etc/rsyslog.d/50-default.conf service rsyslog restart service cron restart more /var/log/cron.log ==== Set locale ==== 1. vi /var/lib/locales/supported.d/en zh_CN.UTF-8 UTF-8 2. locale-gen 3. vi /etc/environment LANG="zh_CN.UTF-8" LANGUAGE="zh_CN:zh:en_US:en" 4. reboot ==== Generate self-signed certificate using OpenSSL ==== cd /etc/ssl mkdir demoCA mkdir demoCA/newcerts touch demoCA/index.txt touch demoCA/serial echo "01" > demoCA/serial openssl genrsa -out server.key 1024 openssl req -new -key server.key -out server.csr -config openssl.cnf -days 3650 # Generate CA openssl req -new -x509 -keyout ca.key -out ca.crt -config openssl.cnf -days 3650 # Sign cert using CA openssl ca -in server.csr -out server.crt -cert ca.crt -keyfile ca.key -config openssl.cnf -days 3650 cat server.crt server.key > server.pem ===== RHEL ===== ==== Change kernel load sequence in RHEL 6.x ==== # Add following parameters behind kernel parameter in /boot/grub/grub.conf rdloaddriver=mptbase,mptspi,mptscsih,vmw_pvscsi rdblacklist=ata_piix