update comment

Use ctrl+d instead of ctrl+c

README.md

@@ -1,7 +1,7 @@
 # Bash-Oneliner
 I am glad that you are here! I was working on bioinformatics a few years ago and was amazed by those single-word bash commands which are much faster than my dull scripts, time saved through learning command-line shortcuts and scripting. Recent years I am working on cloud computing and I keep recording those useful commands here. Not all of them is oneliner, but i put effort on making them brief and swift. I am mainly using Ubuntu, Amazon Linux, RedHat, Linux Mint, Mac and CentOS, sorry if the commands don't work on your system.
 
-This blog will focus on simple bash commands for parsing data and Linux system maintenance that i acquired from work and LPIC exam. I apologize that there are no detailed citation for all the commands, but they are probably from dear Google and Stackoverflow.
+This blog will focus on simple bash commands for parsing data and Linux system maintenance that i acquired from work and LPIC exam. I apologize that there are no detailed citation for all the commands, but they are probably from dear Google and Stack Overflow.
 
 English and bash are not my first language, please correct me anytime, thank you.
 If you know other cool commands, please teach me!
@@ -12,13 +12,13 @@ Here's a more stylish version of [Bash-Oneliner](https://onceupon.github.io/Bash
 
 - [Terminal Tricks](#terminal-tricks)
 - [Variable](#variable)
+- [Math](#math)
 - [Grep](#grep)
 - [Sed](#sed)
 - [Awk](#awk)
 - [Xargs](#xargs)
 - [Find](#find)
 - [Condition and Loop](#condition-and-loop)
-- [Math](#math)
 - [Time](#time)
 - [Download](#download)
 - [Random](#random)
@@ -57,7 +57,7 @@ Esc + u
 Esc + l
 # converts text from cursor to the end of the word to lowercase.
 Esc + c
-# converts letter under the cursor to uppercase.
+# converts letter under the cursor to uppercase, rest of the word to lowercase.
 ```
 ##### Run history number (e.g. 53)
 ```bash
@@ -69,7 +69,6 @@ Esc + c
 !!
 # run the previous command using sudo
 sudo !!
-# of course you need to enter your password
 ```
 
 ##### Run last command and change some parameter using caret substitution (e.g. last command: echo 'aaa' -> rerun as: echo 'bbb')
@@ -98,16 +97,16 @@ sudo !!
 ##### Bash globbing
 ```bash
 # '*' serves as a "wild card" for filename expansion.
-/b?n/?at      #/bin/cat
-
-# '?' serves as a single-character "wild card" for filename expansion.
 /etc/pa*wd    #/etc/passwd
 
-# ‘[]’ serves to match the character from a range.
+# '?' serves as a single-character "wild card" for filename expansion.
+/b?n/?at      #/bin/cat
+
+# '[]' serves to match the character from a range.
 ls -l [a-z]*   #list all files with alphabet in its filename.
 
-# ‘{}’ can be used to match filenames with more than one patterns
+# '{}' can be used to match filenames with more than one patterns
-ls {*.sh,*.py}   #list all .sh and .py files
+ls *.{sh,py}   #list all .sh and .py files
 ```
 
 ##### Some handy environment variables
@@ -135,9 +134,19 @@ $HOSTNAME   current hostname
 ##### Variable substitution within quotes
 ```bash
 # foo=bar
- echo "'$foo'"
+echo $foo
-#'bar'
+# bar
-# double/single quotes around single quotes make the inner single quotes expand variables
+echo "$foo"
+# bar
+# single quotes cause variables to not be expanded
+echo '$foo'
+# $foo
+# single quotes within double quotes will not cancel expansion and will be part of the output
+echo "'$foo'"
+# 'bar'
+# doubled single quotes act as double quotes making variables expand
+echo ''$foo''
+# bar
 ```
 ##### Get the length of variable
 ```bash
@@ -178,11 +187,13 @@ echo ${var[@]#0}
 ```bash
 {var//a/,}
 ```
+
+##### Grep lines with strings from a file (e.g. lines with 'stringA or 'stringB' or 'stringC')
 ```bash
 #with grep
- test="god the father"
+test="stringA stringB stringC"
- grep ${test// /\\\|} file.txt
+grep ${test// /\\\|} file.txt
- # turning the space into 'or' (\|) in grep
+# turning the space into 'or' (\|) in grep
 ```
 
 ##### To change the case of the string stored in the variable to lowercase (Parameter Expansion)
@@ -205,9 +216,9 @@ echo "$bar" # foo
 ```bash
 echo $(( 10 + 5 ))  #15
 x=1
-echo $(( x++ )) #1 , notice that it is still 1, since it's post-incremen
+echo $(( x++ )) #1 , notice that it is still 1, since it's post-increment
 echo $(( x++ )) #2
-echo $(( ++x )) #4 , notice that it is not 3 since it's pre-incremen
+echo $(( ++x )) #4 , notice that it is not 3 since it's pre-increment
 echo $(( x-- )) #4
 echo $(( x-- )) #3
 echo $(( --x )) #1
@@ -265,7 +276,7 @@ echo "var=5;--var"| bc
 #####  Type of grep
 ```bash
 grep = grep -G # Basic Regular Expression (BRE)
-fgrep = grep -F # fixed text, ignoring meta-charachetrs
+fgrep = grep -F # fixed text, ignoring meta-characters
 egrep = grep -E # Extended Regular Expression (ERE)
 pgrep = grep -P # Perl Compatible Regular Expressions (PCRE)
 rgrep = grep -r # recursive
@@ -280,15 +291,15 @@ grep -c "^$"
 ```bash
 grep -o '[0-9]*'
 #or
-grep -oP '\d'
+grep -oP '\d*'
 ```
 #####  Grep integer with certain number of digits (e.g. 3)
 ```bash
-grep ‘[0-9]\{3\}’
+grep '[0-9]\{3\}'
 # or
-grep -E ‘[0-9]{3}’
+grep -E '[0-9]{3}'
 # or
-grep -P ‘\d{3}’
+grep -P '\d{3}'
 ```
 
 #####  Grep only IP address
@@ -395,14 +406,14 @@ grep 'A\|B\|C\|D'
 grep 'A.*B'
 ```
 
-##### Regex any singer character (e.g. ACB or AEB)
+##### Regex any single character (e.g. ACB or AEB)
 ```bash
 grep 'A.B'
 ```
 
 ##### Regex with or without a certain character (e.g. color or colour)
 ```bash
-grep ‘colou?r’
+grep 'colou\?r'
 ```
 
 ##### Grep all content of a fileA from fileB
@@ -456,9 +467,10 @@ sed 1,100d filename
 ##### Remove lines with string (e.g. 'bbo')
 ```bash
 sed "/bbo/d" filename
-- case insensitive:
+# case insensitive:
 sed "/bbo/Id" filename
 ```
+
 ##### Remove lines whose nth character not equal to a value (e.g. 5th character not equal to 2)
 ```bash
 sed -E '/^.{5}[^2]/d'
@@ -510,7 +522,7 @@ sed -i '1s/^/[/' file
 
 ##### Add string at certain line number (e.g. add 'something' to line 1 and line 3)
 ```bash
-sed -e '1isomething -e '3isomething'
+sed -e '1isomething' -e '3isomething'
 ```
 
 ##### Add string to end of file (e.g. "]")
@@ -537,7 +549,7 @@ sed -e 's/$/\}\]/' filename
 sed 's/.\{4\}/&\n/g'
 ```
 
-##### Concatenate/combine/join files with a seperator and next line (e.g separate by ",")
+##### Concatenate/combine/join files with a separator and next line (e.g separate by ",")
 ```bash
 sed -s '$a,' *.json > all.json
 ```
@@ -605,12 +617,12 @@ sed "s/$/\t$i/"
 # $i is the valuable you want to add
 
 # To add the filename to every last column of the file
-for i in $(ls);do sed -i "s/$/\t$i/" $i;done
+for i in $(ls); do sed -i "s/$/\t$i/" $i; done
 ```
 
 ##### Add extension of filename to last column
 ```bash
-for i in T000086_1.02.n T000086_1.02.p;do sed "s/$/\t${i/*./}/" $i;done >T000086_1.02.np
+for i in T000086_1.02.n T000086_1.02.p; do sed "s/$/\t${i/*./}/" $i; done >T000086_1.02.np
 ```
 
 ##### Remove newline\ nextline
@@ -774,7 +786,7 @@ awk '{printf("%s\t%s\n",NR,$0)}'
 
 ##### Break combine column data into rows
 ```bash
-# For example, seperate the following content:
+# For example, separate the following content:
 # David    cat,dog
 # into
 # David    cat
@@ -894,7 +906,7 @@ find /dir/to/A -type f -name "*.py" -print 0| xargs -0 -r -I file cp -v -p file
 
 ##### With sed
 ```bash
-ls |xargs -n1 -I file sed -i '/^Pos/d' filename
+ls |xargs -n1 -I file sed -i '/^Pos/d' file
 ```
 
 ##### Add the file name to the first line of file
@@ -982,6 +994,11 @@ find . -type f -empty
 find . -type f -empty -delete
 ```
 
+##### Recursively count all the files in a directory
+```bash
+find . -type f | wc -l
+```
+
 ## Condition and loop
 [[back to top](#handy-bash-one-liners)]
 
@@ -991,7 +1008,7 @@ find . -type f -empty -delete
 if [[ "$c" == "read" ]]; then outputdir="seq"; else outputdir="write" ; fi
 
 # Test if myfile contains the string 'test':
-if grep -q hello myfile; then …
+if grep -q hello myfile; then echo -e "file contains the string!" ; fi
 
 # Test if mydir is a directory, change to it and do other stuff:
 if cd mydir; then
@@ -1001,9 +1018,12 @@ else
 fi
 
 # if variable is null
-if [ ! -s "myvariable" ]
+if [ ! -s "myvariable" ]; then echo -e "variable is null!" ; fi
 #True of the length if "STRING" is zero.
 
+# Using test command (same as []), to test if the length of variable is nonzero
+test -n "$myvariable" && echo myvariable is "$myvariable" || echo myvariable is not set
+
 # Test if file exist
 if [ -e 'filename' ]
 then
@@ -1017,16 +1037,17 @@ then
 fi
 
 # Test if the value of x is greater or equal than 5
-if [ "$x" -ge 5 ]; then …
+if [ "$x" -ge 5 ]; then echo -e "greater or equal than 5!" ; fi
 
 # Test if the value of x is greater or equal than 5, in bash/ksh/zsh:
-if ((x >= 5)); then …
+if ((x >= 5)); then echo -e "greater or equal than 5!" ; fi
 
 # Use (( )) for arithmetic operation
-if ((j==u+2))
+if ((j==u+2)); then echo -e "j==u+2!!" ; fi
 
 # Use [[ ]] for comparison
-if [[ $age -gt 21 ]]
+if [[ $age -gt 21 ]]; then echo -e "forever 21!!" ; fi
+
 ```
 
 [More if commands](http://tldp.org/LDP/Bash-Beginners-Guide/html/sect_07_01.html)
@@ -1034,7 +1055,7 @@ if [[ $age -gt 21 ]]
 ##### For loop
 ```bash
 # Echo the file name under the current directory
-for i in $(ls); do echo file $i;done
+for i in $(ls); do echo file $i; done
 #or
 for i in *; do echo file $i; done
 
@@ -1042,7 +1063,7 @@ for i in *; do echo file $i; done
 for dir in $(<myfile); do mkdir $dir; done
 
 # Press any key to continue each loop
-for i in $(cat tpc_stats_0925.log |grep failed|grep -o '\query\w\{1,2\}');do cat ${i}.log; read -rsp $'Press any key to continue...\n' -n1 key;done
+for i in $(cat tpc_stats_0925.log |grep failed|grep -o '\query\w\{1,2\}'); do cat ${i}.log; read -rsp $'Press any key to continue...\n' -n1 key; done
 
 # Print a file line by line when a key is pressed,
 oifs="$IFS"; IFS=$'\n'; for line in $(cat myfile); do ...; done
@@ -1052,21 +1073,21 @@ while read -r line; do ...; done <myfile
 for line in $(cat myfile); do echo $line; read -n1; done
 
 #Loop through an array
-for i in "${arrayName[@]}"; do echo $i;done
+for i in "${arrayName[@]}"; do echo $i; done
 
 ```
 
 ##### While loop,
 ```bash
 # Column subtraction of a file (e.g. a 3 columns file)
-while read a b c; do echo $(($c-$b));done < <(head filename)
+while read a b c; do echo $(($c-$b)); done < <(head filename)
 #there is a space between the two '<'s
 
 # Sum up column subtraction
 i=0; while read a b c; do ((i+=$c-$b)); echo $i; done < <(head filename)
 
 # Keep checking a running process (e.g. perl) and start another new process (e.g. python) immediately after it. (BETTER use the wait command! Ctrl+F 'wait')
-while [[ $(pidof perl) ]];do echo f;sleep 10;done && python timetorunpython.py
+while [[ $(pidof perl) ]]; do echo f; sleep 10; done && python timetorunpython.py
 ```
 
 ##### switch (case in bash)
@@ -1105,7 +1126,46 @@ date +%F
 
 # or
 date +'%d-%b-%Y-%H:%M:%S'
-#10-Apr-2020-21:54:40
+# 10-Apr-2020-21:54:40
+
+# Returns the current time with nanoseconds.
+date +"%T.%N"
+# 11:42:18.664217000  
+
+# Get the seconds since epoch (Jan 1 1970) for a given date (e.g Mar 16 2021)
+date -d "Mar 16 2021" +%s
+# 1615852800
+# or
+date -d "Tue Mar 16 00:00:00 UTC 2021"  +%s
+# 1615852800  
+
+# Convert the number of seconds since epoch back to date
+date --date @1615852800
+# Tue Mar 16 00:00:00 UTC 2021
+
+```
+
+##### Print current time point for N days ago or N days after
+```bash
+# print current date first (for the following example)
+date +"%F %H:%M:%S"
+# 2023-03-11 16:17:09
+
+# print the time that is 1 day ago
+date -d"1 day ago" +"%F %H:%M:%S"
+# 2023-03-10 16:17:09
+
+# print the time that is 7 days ago
+date -d"7 days ago" +"%F %H:%M:%S"
+# 2023-03-04 16:17:09
+
+# print the time that is a week ago
+date -d"1 week ago" +"%F %H:%M:%S"
+# 2023-03-04 16:17:09
+
+# add 1 day to date
+date -d"-1 day ago" +"%F %H:%M:%S"
+# 2023-03-12 16:17:09
 ```
 
 ##### wait for random duration (e.g. sleep 1-5 second, like adding a jitter)
@@ -1218,7 +1278,7 @@ shuf -n 100 filename
 
 ##### Random order (lucky draw)
 ```bash
-for i in a b c d e; do echo $i; done| shuf
+for i in a b c d e; do echo $i; done | shuf
 ```
 
 ##### Echo series of random numbers between a range (e.g. shuffle numbers from 0-100, then pick 15 of them randomly)
@@ -1707,6 +1767,18 @@ du -h
 du -sk /var/log/* |sort -rn |head -10
 ```
 
+##### check the Inode utilization
+```
+df -i
+# Filesystem      Inodes  IUsed   IFree IUse% Mounted on
+# devtmpfs        492652    304  492348    1% /dev
+# tmpfs           497233      2  497231    1% /dev/shm
+# tmpfs           497233    439  496794    1% /run
+# tmpfs           497233     16  497217    1% /sys/fs/cgroup
+# /dev/nvme0n1p1 5037976 370882 4667094    8% /
+# tmpfs           497233      1  497232    1% /run/user/1000
+```
+
 ##### Show all file system type
 ```bash
 df -TH
@@ -2021,7 +2093,7 @@ http://onceuponmine.blogspot.tw/2017/07/create-your-first-simple-daemon.html
 ##### Tutorial for using your gmail to send email
 http://onceuponmine.blogspot.tw/2017/10/setting-up-msmtprc-and-use-your-gmail.html
 
- ##### Using telnet to test open ports, test if you can connect to a port (e.g 53) of a server (e.g 192.168.2.106)
+##### Using telnet to test open ports, test if you can connect to a port (e.g 53) of a server (e.g 192.168.2.106)
 ```bash
 telnet 192.168.2.106 53
 ```
@@ -2163,6 +2235,7 @@ sar -n ALL
 
 # reading SAR log file using -f
 sar -f /var/log/sa/sa31|tail
+```
 
 ##### Reading from journal file
 ```bash
@@ -2380,7 +2453,7 @@ nc -vw5 google.com 22
 # From server A:
 $ sudo nc -l 80
 # then you can connect to the 80 port from another server (e.g. server B):
-# e.g. telent <server A IP address> 80
+# e.g. telnet <server A IP address> 80
 # then type something in server B
 # and you will see the result in server A!
 ```
@@ -2522,16 +2595,16 @@ sudo iptables –A INPUT –s <IP> -p tcp –dport 80 –j DROP
 ```bash
 # If file is not specified, the file /usr/share/dict/words is used.
 look phy|head -n 10
-# Phil
+# phycic
-# Philadelphia
+# Phyciodes
-# Philadelphia's
+# phycite
-# Philby
+# Phycitidae
-# Philby's
+# phycitol
-# Philip
+# phyco-
-# Philippe
+# phycochrom
-# Philippe's
+# phycochromaceae
-# Philippians
+# phycochromaceous
-# Philippine
+# phycochrome
 ```
 
 ##### Repeat printing string n times (e.g. print 'hello world' five times)
@@ -2586,7 +2659,41 @@ sdiff fileA fileB
 
 ##### Compare two files, strip trailing carriage return/ nextline (e.g. fileA, fileB)
 ```bash
- diff fileA fileB --strip-trailing-cr
+diff fileA fileB --strip-trailing-cr
+```
+
+##### Find common/differing lines
+```bash
+# having two sorted and uniqed files (for example after running `$ sort -uo fileA fileA` and same for fileB):
+# ------
+# fileA:
+# ------
+# joey
+# kitten
+# piglet
+# puppy
+# ------
+# fileB:
+# ------
+# calf
+# chick
+# joey
+# puppy
+#
+# Find lines in both files
+comm -12 fileA fileB
+# joey
+# puppy
+#
+# Find lines in fileB that are NOT in fileA
+comm -13 fileA fileB
+# calf
+# chick
+#
+# Find lines in fileA that are NOT in fileB
+comm -23 fileA fileB
+# kitten
+# piglet
 ```
 
 ##### Number a file (e.g. fileA)
@@ -2662,8 +2769,8 @@ echo {1,2}{1,2}
 ```bash
 set = {A,T,C,G}
 group= 5
-for ((i=0; i<$group; i++));do
+for ((i=0; i<$group; i++)); do
-    repetition=$set$repetition;done
+    repetition=$set$repetition; done
     bash -c "echo "$repetition""
 ```
 
@@ -2753,12 +2860,28 @@ var=$((var+1))
 cat filename|rev|cut -f1|rev
 ```
 
-##### Cat to a file
+##### Create or replace a file with contents
 ```bash
 cat >myfile
 let me add sth here
-exit by control + c
+# exit with ctrl+d
-^C
+
+# or using tee
+tee myfile
+let me add sth else here
+# exit with ctrl+d
+```
+
+##### Append to a file with contents
+```bash
+cat >>myfile
+let me add sth here
+# exit with ctrl+d
+
+# or using tee
+tee -a myfile
+let me add sth else here
+# exit with ctrl+d
 ```
 
 ##### Clear the contents of a file (e.g. filename)
@@ -2841,7 +2964,7 @@ tac filename
 
 ##### Reverse the result from `uniq -c`
 ```bash
-while read a b; do yes $b |head -n $a ;done <test.txt
+while read a b; do yes $b |head -n $a ; done <test.txt
 ```
 
 
@@ -2877,6 +3000,13 @@ cal
 # only display November
 cal -m 11
 ```
+
+##### Convert the hexadecimal MD5 checksum value into its base64-encoded format.
+```bash
+openssl md5 -binary /path/to/file| base64
+# NWbeOpeQbtuY0ATWuUeumw==
+```
+
 ##### Forces applications to use the default language for output
 ```bash
 export LC_ALL=C
@@ -3098,7 +3228,7 @@ scp -r directoryname user@ip:/path/to/send
 echo $?
 ```
 
-##### Extract .xf
+##### Extract .xz
 ```
 unxz filename.tar.xz
 # then
@@ -3138,8 +3268,7 @@ yes n
 # or 'anything':
 yes anything
 
-# For example:
+# pipe yes to other command
-```bash
 yes | rm -r large_directory
 ```
 
@@ -3150,9 +3279,9 @@ fallocate -l 10G 10Gigfile
 
 ##### Create dummy file of certain size (e.g. 200mb)
 ```bash
-dd if=/dev/zero of=//dev/shm/200m bs=1024k count=200
+dd if=/dev/zero of=/dev/shm/200m bs=1024k count=200
 # or
-dd if=/dev/zero of=//dev/shm/200m bs=1M count=200
+dd if=/dev/zero of=/dev/shm/200m bs=1M count=200
 
 # Standard output:
 # 200+0 records in
@@ -3165,9 +3294,29 @@ dd if=/dev/zero of=//dev/shm/200m bs=1M count=200
 watch -n 1 wc -l filename
 ```
 
+##### Use Bash Strict Mode
+```bash
+# These options can make your code safer but, depending on how your pipeline is written, it might be too aggressive 
+# or it might not catch the errors that you are interested in
+
+# for reference see https://gist.github.com/mohanpedala/1e2ff5661761d3abd0385e8223e16425
+#               and https://mywiki.wooledge.org/BashPitfalls#set_-euo_pipefail
+
+set -o errexit      # exit immediately if a pipeline returns a non-zero status
+set -o errtrace     # trap ERR from shell functions, command substitutions, and commands from subshell
+set -o nounset      # treat unset variables as an error
+set -o pipefail     # pipe will exit with last non-zero status, if applicable
+set -Eue -o pipefail  # shorthand for above (pipefail has no short option)
+```
+
 ##### Print commands and their arguments when execute (e.g. echo `expr 10 + 20 `)
 ```bash
 set -x; echo `expr 10 + 20 `
+# or
+set -o xtrace; echo `expr 10 + 20 `
+
+# to turn it off..
+set +x
 ```
 
 ##### Print some meaningful sentences to you (install fortune first)
@@ -3198,6 +3347,9 @@ q -d "," "select c3,c4,c5 from /path/to/file.txt where c3='foo' and c5='boo'"
 # Create session and attach:
 screen
 
+# Create a screen and name it 'test'
+screen -S test
+
 # Create detached session foo:
 screen -S foo -d -m
 
@@ -3216,15 +3368,16 @@ screen -r foo
 # Kill session foo:
 screen -r foo -X quit
 
+
 # Scroll:
-Hit your screen prefix combination (C-a / control+A), then hit Escape.
+# Hit your screen prefix combination (C-a / control+A), then hit Escape.
-Move up/down with the arrow keys (↑ and ↓).
+# Move up/down with the arrow keys (↑ and ↓).  
 
 # Redirect output of an already running process in Screen:
- (C-a / control+A), then hit 'H'
+# (C-a / control+A), then hit 'H'  
 
 # Store screen output for Screen:
-Ctrl+A, Shift+H
+# Ctrl+A, Shift+H  
 # You will then find a screen.log file under current directory.  
 ```
 
@@ -3351,6 +3504,8 @@ source .venv/bin/activate
 type pip
 # 5. Now you can install your pip package, here requirements.txt is simply a txt file containing all the packages you want. (e.g tornado==4.5.3).
 pip install -r requirements.txt
+# 6. Exit virtual environment
+deactivate
 ```