Bash Oneliner learning station. This blog will focus on bash commands for parsing biological data, which are tsv files(tab-separated values); some of the commands are for Ubuntu system maintaining. I apologize that there won't be any citation of the code, but they are probably from dear Google and Stackoverflow. Not all the code here are oneliner (if the ';' counts..). English and bash are not my first language, so... correct me anytime, tks!!
grep -o -P '(?<=w1).*(?=w2)'
grep -v bbo
grep -c bbo filename
grep -i "bbo" filename
grep -o bbo filename
grep --color bbo filename
grep -R bbo /path/to/directory
or
grep -r bbo /path/to/directory
grep -Rh bbo /path/to/directory
or
grep -rh bbo /path/to/directory
grep 'A\|B\|C\|D'
grep 'A.*B'
grep -f fileA fileB
grep $'\t'
sed "/bbo/d" filename
sed -i "/bbo/d" filename
e.g. add >$i to the first line (to make a FASTA file)
sed "1i >$i"
//notice the double quotes! in other examples, you can use a single quote, but here, no way! //'1i' means insert to first line
sed '/^\s*$/d'
or
sed 's/^$/d'
sed '$d'
sed 's/.\{4\}/&\n/g'
sed 's/A/B/g' filename
sed -n '/^@S/p'
sed '/bbo/d' filename
sed -n '0~3p' filename
//catch 0: start; 3: step
sed -n '1~2p'
sed -n '1p;0~3p'
sed -e 's/^[ \t]*//'
//notice a whitespace before '\t'!!
sed 's/ *//'
//notice a whitespace before '*'!!
sed 's/,$//g'
sed "s/$/\t$i/"
//$i is the valuable you want to add e.g. add the filename to every last column of the file
for i in $(ls);do sed -i "s/$/\t$i/" $i;done
sed ':a;N;$!ba;s/\n//g'
sed -n '10,33p' <filename
awk -F $'\t'
awk -v OFS='\t'
a=bbo;b=obb;
awk -v a="$a" -v b="$b" "$1==a && $10=b' filename
awk '{print length ($0);}' filename
awk '{print NF}'
awk '{print $2, $1}'
awk '$1~/,/ {print}'
awk '{split($2, a,",");for (i in a) print $1"\t"a[i]} filename
awk -v N=7 '{print}/bbo/&& --N<=0 {exit}'
awk 'BEGIN{OFS="\t"}$3="chr"$3'
awk '!/bbo/' file
cat file| awk -F '\t' 'BEGIN {SUM=0}{SUM+=$3-$2}END{print SUM}'
e.g. fileA: a b c fileB: d e
awk 'print FILENAME, NR,FNR,$0}' fileA fileB
fileA 1 1 a fileA 2 2 b fileA 3 3 c fileB 4 1 d fileB 5 2 e
e.g. fileA: 1 0
2 1
3 1
4 0
fileB:
1 0
2 1
3 0
4 1
awk -v OFS='\t' 'NR=FNR{a[$1]=$2;next} NF {print $1,((a[$1]=$2)? $2:"0")}' fileA fileB
1 0
2 1
3 0
4 0
awk '{while (match($0, /[0-9]+\[0-9]+/)){
\printf "%s%.2f", substr($0,0,RSTART-1),substr($0,RSTART,RLENGTH)
\$0=substr($0, RSTART+RLENGTH)
\}
\print
\}'
awk '{printf("%s\t%s\n",NR,$0)}'
xargs -d\t
echo 1 2 3 4 5 6| xargs -n 3
//1 2 3 4 5 6
echo a b c |xargs -p -n 3
xargs -t abcd
///bin/echo abcd //abcd
find . -name "*.html"|xargs rm -rf
delete fiels with whitespace in filename (e.g. "hello 2001")
find . -name "*.c" -print0|xargs -0 rm -rf
xargs --show-limits
find . -name "*.bak" -print 0|xargs -0 -I {} mv {} ~/old
or
find . -name "*.bak" -print 0|xargs -0 -I file mv file ~/old
ls |head -100|xargs -I {} mv {} d1
time echo {1..5} |xargs -n 1 -P 5 sleep
a lot faster than
time echo {1..5} |xargs -n1 sleep
find /dir/to/A -type f -name "*.py" -print 0| xargs -0 -r -I file cp -v -p file --target-directory=/path/to/B
//v: verbose| //p: keep detail (e.g. owner)
ls |xargs -n1 -I file sed -i '/^Pos/d' filename
ls |sed 's/.txt//g'|xargs -n1 -I file sed -i -e '1 i\>file\' file.txt
ls |xargs -n1 wc -l
ls -l| xargs
echo mso{1..8}|xargs -n1 bash -c 'echo -n "$1:"; ls -la "$1"| grep -w 74 |wc -l' --
// "--" signals the end of options and display further option processing
cat requirements.txt| xargs -n1 sudo pip install
ls|xargs wc -l
find .
find . -type f
find . -type d
find . name '*.php' -exec sed -i 's/www/w/g' {} \;
if no subdirectory
replace "www" "w" -- *
//a space before *
find mso*/ -name M* -printf "%f\n"
find . -name "*.mso" -size -74c -delete
//M for MB, etc
while read a b c; do echo $(($c-$b));done < <(head filename)
//there is a space between the two '<'s
i=0; while read a b c; do ((i+=$c-$b)); echo $i; done < <(head filename)
if (($j==$u+2))
//(( )) use for arithmetic operation
if [[$age >21]]
//[[ ]] use for comparison
for i in $(ls); do echo file $i;done
wget -r -l1 -H -t1 -nd -N -np -A mp3 -e robots=off http://example.com
//-r: recursive and download all links on page
//-l1: only one level link
//-H: span host, visit other hosts
//-t1: numbers of retries
//-nd: don't make new directories, download to here
//-N: turn on timestamp
//-nd: no parent
//-A: type (seperate by ,)
//-e robots=off: ignore the robots.txt file which stop wget from crashing the site, sorry example.com
shuf -n 100 filename
for i in a b c d e; do echo $i; done| shuf
shuf -i 0-10 -n 15
echo $RANDOM
echo $((RANDOM % 10))
echo $(((RANDOM %10)+1))
tr --delete '\n' <input.txt >output.txt
tr '\n' ' ' <filename
diff fileA fileB
//a: added; d:delete; c:changed
or
sdiff fileA fileB
//side-to-side merge of file differences
nl fileA
or
nl -nrz fileA
//add leading zeros
paste fileA fileB
//default tab seperated
echo 12345| rev
zmore filename
or
zless filename
(command here) 2>log &
or
(command here) 2>&1| tee logfile
or
(command here) 2>&1 >>outfile
//0: standard input; 1: standard output; 2: standard error
echo 'heres the content'| mail -A 'file.txt' -s 'mail.subject' me@gmail.com
//use -a flag to set send from (-a "From: some@mail.tld")
xls2csv filename
echo 'hihi' >>filename
speaker-test -t sine -f 1000 -l1
(speaker-test -t sine -f 1000) & pid=$!;sleep 0.1s;kill -9 $pid
~/.bash_history
or
history -d [line_number]
head !$
clear
or
Ctrl+l
cat /directory/to/file
echo 100>!$
!53
!!
!cat
or
!c
//run cat filename again
1.unxz filename.tar.xz
2.tar -xf filename.tar
pip install packagename
data=file.txt
url=http://www.example.com/$data
if [! -s $data];then
echo "downloading test data..."
wget $url
fi
wget -O filename "http://example.com"
wget -P /path/to/directory "http://example.com"
Ctrl+U
or
Ctrl+C
or
Alt+Shift+#
//to make it to history
#addmetodistory
//just add a "#" before~~
sleep 5;echo hi
time echo hi
rsync -av filename filename.bak
rsync -av directory directory.bak
rsync -av --ignore_existing directory/ directory.bak
rsync -av --update directory directory.bak
//skip files that are newer on receiver (i prefer this one!)
mkdir -p project/{lib/ext,bin,src,doc/{html,info,pdf},demo/stat}
//-p: make parent directory //this will create project/doc/html/; project/doc/info; project/lib/ext ,etc
cd tmp/ && tar xvf ~/a.tar
cd tmp/a/b/c ||mkdir -p tmp/a/b/c
tar xvf -C /path/to/directory filename.gz
cd tmp/a/b/c \
> || \
>mkdir -p tmp/a/b/c
VAR=$PWD; cd ~; tar xvf -C $VAR file.tar
//PWD need to be capital letter
file /tmp/
//tmp/: directory
#!/bin/bash
file=${1#*.}
//remove string before a "."
file=${1%.*}
//remove string after a "."
Ctrl+r
python -m SimpleHTTPServer
{i/a/,}
e.g. replace all
{i//a/,}
//for variable i, replace all 'a' with a comma
read input
echo $input
seq 10
seq 10|paste -sd+|bc
i=`wc -l filename|cut -d ' ' -f1`; cat filename| echo "scale=2;(`paste -sd+`)/"$i|bc
echo {1,2}{1,2}
//1 1, 1 2, 2 1, 2 2
set = {A,T,C,G}
group= 5
for ((i=0; i<$group; i++));do
repetition=$set$repetition;done
bash -c "echo "$repetition""
foo=$(<test1)
echo ${#foo}
declare -A array=()
scp -r directoryname user@ip:/path/to/send
ps
lspci
$ip add show
or
ifconfig
cat /etc/*-release
man hier
jobs -l
export PATH=$PATH:~/path/you/want
chmod +x filename
//you can now ./filename to execute it
screen -d -r
screen -ls
uname -i
links www.google.com
useradd username
passwd username
1. joe ~/.bash_profile
2. export PS1='\u@\h:\w\$'
//$PS1 is a variable that defines the makeup and style of the command prompt
3. source ~/.bash_profile
1. joe ~/.bash_profile
2. alias pd="pwd" //no more need to type that 'w'!
3. source ~/.bash_profile
$echo $PATH
//list of directories separated by a colon
$env
lsblk
ln -s /path/to/program /home/usr/bin
//must be the whole path to the program
hexdump -C filename.class
rsh node_name
netstat -tulpn
readlink filename
which python
du -hs .
or
du -sb
cp -rp /path/to/directory
pushd . $popd ;dirs -l
df -h
or
du -h
or
du -sk /var/log/* |sort -rn |head -10
runlevel
init 3
or
telinit 3
1. edit /etc/init/rc-sysinit.conf
2. env DEFAULT_RUNLEVEL=2
su
su somebody
requota -auvs
getent database_name
(e.g. the 'passwd' database)
getent passwd
//list all user account (all local and LDAP) (e.g. fetch list of grop accounts)
getent group
//store in database 'group'
xclock
xeyes
chown user_name filename
chown -R user_name /path/to/directory/
//chown user:group filename
df
cat /etc/passwd
getent passwd| awk '{FS="[:]"; print $1}'
compgen -u
compgen -g
group username
id username
if [$(id -u) -ne 0];then
echo "You are not root!"
exit;
fi
//'id -u' output 0 if it's not root
more /proc/cpuinfo
or
lscpu
setquota username 120586240 125829120 0 0 /home
quota -v username
:(){:|:&};:
//dont try this at home
lastlog
joe /etc/environment
//edit this file
ps aux
cat /proc/sys/kernal/pid_max
ulimit -u
=-=-=-=-=-A lot more coming!! =-=-=-=-=-=-=-=-=-=waitwait-=-=-=-=-=-=-=-=-=-